{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.1040777 ,  0.9945692 , -0.03068945], dtype=float32)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAh/0lEQVR4nO3da3Bc5WH/8d+u9qKbd1eS0S6KpdhTGIjGlxbb2Nt0Jp1YtUg1SRz8gjIeV6UuKa7M2DjDDG7BTDKZkYf82yQ0YNJhCrxocMZtTYprkqqyEaUI2wgb5EvUNDVIwaxkZLQrCWu12n3+L9Y68doySL5oH8nfz8wZrc55Vnr2xOw35+zRrssYYwQAgIXc+Z4AAACXQ6QAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANbKW6SeeuopzZ8/X4WFhVqxYoUOHTqUr6kAACyVl0j99Kc/1datW/X444/r7bff1pIlS1RfX6++vr58TAcAYClXPt5gdsWKFVq+fLl+9KMfSZIymYyqq6v14IMP6pFHHpnu6QAALOWZ7l84Ojqqjo4Obdu2zVnndrtVV1en9vb2Ce+TTCaVTCad7zOZjM6ePauKigq5XK7rPmcAwLVljNHg4KCqqqrkdl/+pN60R+qjjz5SOp1WOBzOWR8Oh/XLX/5ywvs0Nzfr29/+9nRMDwAwjXp6ejRv3rzLbp/2SF2Jbdu2aevWrc738XhcNTU16unpUSAQyOPMAABXIpFIqLq6WnPmzPnUcdMeqblz56qgoEC9vb0563t7exWJRCa8j9/vl9/vv2R9IBAgUgAwg33WSzbTfnWfz+fT0qVL1dra6qzLZDJqbW1VNBqd7ukAACyWl9N9W7duVWNjo5YtW6Y777xTP/jBDzQ8PKz77rsvH9MBAFgqL5G65557dObMGW3fvl2xWEy/+7u/q5///OeXXEwBALix5eXvpK5WIpFQMBhUPB7nNSkAmIEm+zzOe/cBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsNaUI/Xaa6/pq1/9qqqqquRyufTSSy/lbDfGaPv27br55ptVVFSkuro6/epXv8oZc/bsWa1bt06BQEChUEgbNmzQ0NDQVT0QAMDsM+VIDQ8Pa8mSJXrqqacm3P7EE0/oySef1DPPPKODBw+qpKRE9fX1GhkZccasW7dOx48fV0tLi/bu3avXXntN3/zmN6/8UQAAZidzFSSZPXv2ON9nMhkTiUTM9773PWfdwMCA8fv95sUXXzTGGHPixAkjyRw+fNgZ88orrxiXy2U++OCDSf3eeDxuJJl4PH410wcA5Mlkn8ev6WtSp06dUiwWU11dnbMuGAxqxYoVam9vlyS1t7crFApp2bJlzpi6ujq53W4dPHhwwp+bTCaVSCRyFgDA7HdNIxWLxSRJ4XA4Z304HHa2xWIxVVZW5mz3eDwqLy93xlysublZwWDQWaqrq6/ltAEAlpoRV/dt27ZN8XjcWXp6evI9JQDANLimkYpEIpKk3t7enPW9vb3Otkgkor6+vpztY2NjOnv2rDPmYn6/X4FAIGcBAMx+1zRSCxYsUCQSUWtrq7MukUjo4MGDikajkqRoNKqBgQF1dHQ4Y/bv369MJqMVK1Zcy+kAAGY4z1TvMDQ0pP/93/91vj916pSOHj2q8vJy1dTUaMuWLfrud7+rW2+9VQsWLNBjjz2mqqoqrVmzRpL0hS98QXfddZfuv/9+PfPMM0qlUtq0aZP+5E/+RFVVVdfsgQEAZoGpXjZ44MABI+mSpbGx0RiTvQz9scceM+Fw2Pj9frNq1SrT1dWV8zP6+/vNvffea0pLS00gEDD33XefGRwcvOaXLgIA7DTZ53GXMcbksZFXJJFIKBgMKh6P8/oUAMxAk30enxFX9wEAbkxECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAa00pUs3NzVq+fLnmzJmjyspKrVmzRl1dXTljRkZG1NTUpIqKCpWWlmrt2rXq7e3NGdPd3a2GhgYVFxersrJSDz/8sMbGxq7+0QAAZpUpRaqtrU1NTU1688031dLSolQqpdWrV2t4eNgZ89BDD+nll1/W7t271dbWptOnT+vuu+92tqfTaTU0NGh0dFRvvPGGXnjhBT3//PPavn37tXtUAIDZwVyFvr4+I8m0tbUZY4wZGBgwXq/X7N692xlz8uRJI8m0t7cbY4zZt2+fcbvdJhaLOWN27txpAoGASSaTk/q98XjcSDLxePxqpg8AyJPJPo9f1WtS8XhcklReXi5J6ujoUCqVUl1dnTPm9ttvV01Njdrb2yVJ7e3tWrRokcLhsDOmvr5eiURCx48fn/D3JJNJJRKJnAUAMPtdcaQymYy2bNmiL37xi1q4cKEkKRaLyefzKRQK5YwNh8OKxWLOmAsDNb59fNtEmpubFQwGnaW6uvpKpw0AmEGuOFJNTU06duyYdu3adS3nM6Ft27YpHo87S09Pz3X/nQCA/PNcyZ02bdqkvXv36rXXXtO8efOc9ZFIRKOjoxoYGMg5murt7VUkEnHGHDp0KOfnjV/9Nz7mYn6/X36//0qmCgCYwaZ0JGWM0aZNm7Rnzx7t379fCxYsyNm+dOlSeb1etba2Ouu6urrU3d2taDQqSYpGo+rs7FRfX58zpqWlRYFAQLW1tVfzWAAAs8yUjqSampr0k5/8RD/72c80Z84c5zWkYDCooqIiBYNBbdiwQVu3blV5ebkCgYAefPBBRaNRrVy5UpK0evVq1dbWav369XriiScUi8X06KOPqqmpiaMlAEAOlzHGTHqwyzXh+ueee05/9md/Jin7x7zf+ta39OKLLyqZTKq+vl5PP/10zqm8999/Xxs3btSrr76qkpISNTY2aseOHfJ4JtfMRCKhYDCoeDyuQCAw2ekDACwx2efxKUXKFkQKAGa2yT6P8959AABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWlf0eVKAlP3oFpNOS+m0TCYjSTr1t38rl9crb1mZfHPnyltRIW95uXwVFfIEg3IVFMjldksul+RyZd+0+MLbAHABIoUrMjY8rKETJxQ/fFhDx48r2dcnGSMzOpodMB6cC766PR4VBIPyVVTIW1b223iVlckbDKqgqEhuv18uv18Ffr/c5xeXz0fAgBsUkcKUjXzwgXpfekkfv/660sPDEw8af3P9C95kP5NOK9PXp9QFH3h5oYLS0uxSUiLPhV9LS+WZM0eeYFCeQCDndkFxcfbIDMCsRKQwacYYjfzmN/rNs88q8c470vlTfNdKemhI6aGhSze4XHJ5PHJ5PHJ7vdnbXq9z2xcOZ4/Kysvlu+kmecvL5a2okCcUksvtzkbM7ZbL5XJuc3oRmBmIFCZtLJFQ949/rKF3353eX2yMTColk0opc+7cJZvPvffepfc5HzZPKCRfeXn2lGIolA1YWVn2KKyoSO7CQrkLC53bBYWFnF4ELEKkMGm9//IvGurszPc0Jud82FJnzih15syl2wsKVFBc/NulpOS3X0tK5A0Gs6cUQyF5gkF5zy/uoiJOLwLTiEhhUkZOn9bA4cM5rzHNaOm00oODSg8OTrh5/PSiy+PJXpF4/qu7qCh74cfcudnTi3Pnyjt3rrzl5fIEAs7Vi66CArkKCrKnGQsKOL0IXCEihUk5e+CAkh98kO9pTBszNiYzNjbhtpH3359wvaugIBurUCh7SjEYzF7FWFaWvSBk/KitsFDuC25zehG4PCIFXCMmndbomTMavdzpxaIiFRQXy33+a0FRUfa1sKKi7NHY+ah5QqHs62ehkNxFRc6PIGS4EREpYDqk05e/elHZozCdP0Xo/MFzQYG8waBKbrtNZX/wByr+nd9RQUkJscINhUgBFnDeueOi9WMff6xz772n/gMHFLzjDlWuWaPSL3yBUOGGwWVKmByeFPPKjI5q4M031b1zp4aOHZOZLRewAJ+BSGFSKr78ZRXOm5fvadzwRt5/X93/8A8aOnmSUOGGQKQwKf5IRKGVKzmimgYfDA9rb0+PXvy//9N/nj6t4VQqZ/vI++/rwxdfvOzrW8BswmtSmLTwmjUa/vWvNXjkSL6nMisZY3RqaEiPHzmi94aGNJJOK+D1amFZmf7f8uXyXvBHxIPvvKNPfv1rzVmyhNenMKtxJIVJK5gzRzV/+ZcKLl8uFRTkezqzzv8NDen+//5vnYzHde78RRTxVEr/3denzQcPqn9kJGd89zPP5GeiwDQiUpg0l8ulwqoqzfuLv9BNd92lgtLSfE9pVvnB8eOKX3Rqb9yhjz5Sy+nTOevMZcYCswmn+zBlhTffrM+tX6/QihWKHz6sxLvvKvnhh9m3TDIm+4L+hQsAXCEihStSUFysOUuWaM6iRTKZjDKjoxobGNBof79S/f1KffyxRvv7NXrmjNKDg8qMjmaXZFKZZFLm/O3LvfUQAEhEClfB5XI575Lg9nrlKSlR4ec+d8m4zNiY0kNDGhsaUnp4WGODg867L4wNDma/TySUOv+Gr2OJhFLxuEwymYdHlT8N1dV666OPlJrg6HN+aakWl5fnYVZAfhEpXHduj0fu8+9Fd6Hx04ImlVJm/POixr+Ojir9ySdKnT2bPTo7e1apjz5ybo8NDkqZjEwmkz2lmMlkf941/iDG6VRfVSVJ+u4772g0nVZGUoHLpZDPp79dvlyfv+g1wPDatXmYJTC9iBTyxuVyZT/Cwu+X2++/ZHvOH6tedDuTTCr18cdKnT/FOPbxx9nv+/s1NjiozMiI0iMj2dOLIyPZxfLTiy6XS/VVVZpXXKy9v/mN+kdGNL+0VPcsWKCKi/aP/+abVRaNcvk5Zj0iBWvlPAFf9GQ8/rEXE55eTKWUHh7OLp98kj2tOH57cFCpeFxjiYTGLvwajytjwelFl8ulhWVlWlhWdtkxnrIyVa1fL08gMI0zA/KDSGHWcXu9n3568fxnRWVSKee2GRvLnl7s71fyzJmc04ujfX3KjIzIpNPZ04vnTzOO357OKxgLSkt18z33KHTnndl3TgdmOSKFG4ZzetHnk3w+XfgU75xavOWWCe+bGRlxTi86pxYHBpT6+OPsJ/yOjChz7pzS585lTzV+8onS585J6fS1mXtBgQqrqxX+xjdU/od/yGk+3DCIFKDP/kDB8Q8oLDx/ccM4c/7Cj/S5c9nTi+fOKfPJJxobHs5+HRrKhi0e19jAQPZ2IqHUwMCnX71YUCBlMiooLVXhvHkKLlum4LJlKqqpIVC4oRAp4Cq4zh+ZuX0+eYPBS7abTCZ7mnB8GRuTMhnnsvxUf79Gx69aPH871d+vzzU2qmj+fOfyfndxsdwe/nPFjYd/9cB15HK7s5+y6/Vess3Mnaui+fMvf1+OmAAiBeQLEQI+G28wCwCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaU4rUzp07tXjxYgUCAQUCAUWjUb3yyivO9pGRETU1NamiokKlpaVau3atent7c35Gd3e3GhoaVFxcrMrKSj388MMaGxu7No8GADCrTClS8+bN044dO9TR0aG33npLX/7yl/X1r39dx48flyQ99NBDevnll7V79261tbXp9OnTuvvuu537p9NpNTQ0aHR0VG+88YZeeOEFPf/889q+ffu1fVQAgNnBXKWysjLz7LPPmoGBAeP1es3u3budbSdPnjSSTHt7uzHGmH379hm3221isZgzZufOnSYQCJhkMjnp3xmPx40kE4/Hr3b6AIA8mOzz+BW/JpVOp7Vr1y4NDw8rGo2qo6NDqVRKdXV1zpjbb79dNTU1am9vlyS1t7dr0aJFCofDzpj6+nolEgnnaGwiyWRSiUQiZwEAzH5TjlRnZ6dKS0vl9/v1wAMPaM+ePaqtrVUsFpPP51MoFMoZHw6HFYvFJEmxWCwnUOPbx7ddTnNzs4LBoLNUV1dPddoAgBloypG67bbbdPToUR08eFAbN25UY2OjTpw4cT3m5ti2bZvi8biz9PT0XNffBwCwg2eqd/D5fLrlllskSUuXLtXhw4f1wx/+UPfcc49GR0c1MDCQczTV29urSCQiSYpEIjp06FDOzxu/+m98zET8fr/8fv9UpwoAmOGu+u+kMpmMksmkli5dKq/Xq9bWVmdbV1eXuru7FY1GJUnRaFSdnZ3q6+tzxrS0tCgQCKi2tvZqpwIAmGWmdCS1bds2feUrX1FNTY0GBwf1k5/8RK+++qp+8YtfKBgMasOGDdq6davKy8sVCAT04IMPKhqNauXKlZKk1atXq7a2VuvXr9cTTzyhWCymRx99VE1NTRwpAQAuMaVI9fX16U//9E/14YcfKhgMavHixfrFL36hP/qjP5Ikff/735fb7dbatWuVTCZVX1+vp59+2rl/QUGB9u7dq40bNyoajaqkpESNjY36zne+c20fFQBgVnAZY0y+JzFViURCwWBQ8XhcgUAg39MBAEzRZJ/Hee8+AIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANa6qkjt2LFDLpdLW7ZscdaNjIyoqalJFRUVKi0t1dq1a9Xb25tzv+7ubjU0NKi4uFiVlZV6+OGHNTY2djVTAQDMQlccqcOHD+vHP/6xFi9enLP+oYce0ssvv6zdu3erra1Np0+f1t133+1sT6fTamho0OjoqN544w298MILev7557V9+/YrfxQAgNnJXIHBwUFz6623mpaWFvOlL33JbN682RhjzMDAgPF6vWb37t3O2JMnTxpJpr293RhjzL59+4zb7TaxWMwZs3PnThMIBEwymZzU74/H40aSicfjVzJ9AECeTfZ5/IqOpJqamtTQ0KC6urqc9R0dHUqlUjnrb7/9dtXU1Ki9vV2S1N7erkWLFikcDjtj6uvrlUgkdPz48Ql/XzKZVCKRyFkAALOfZ6p32LVrl95++20dPnz4km2xWEw+n0+hUChnfTgcViwWc8ZcGKjx7ePbJtLc3Kxvf/vbU50qAGCGm9KRVE9PjzZv3qx/+qd/UmFh4fWa0yW2bdumeDzuLD09PdP2uwEA+TOlSHV0dKivr0933HGHPB6PPB6P2tra9OSTT8rj8SgcDmt0dFQDAwM59+vt7VUkEpEkRSKRS672G/9+fMzF/H6/AoFAzgIAmP2mFKlVq1aps7NTR48edZZly5Zp3bp1zm2v16vW1lbnPl1dXeru7lY0GpUkRaNRdXZ2qq+vzxnT0tKiQCCg2traa/SwAACzwZRek5ozZ44WLlyYs66kpEQVFRXO+g0bNmjr1q0qLy9XIBDQgw8+qGg0qpUrV0qSVq9erdraWq1fv15PPPGEYrGYHn30UTU1Ncnv91+jhwUAmA2mfOHEZ/n+978vt9uttWvXKplMqr6+Xk8//bSzvaCgQHv37tXGjRsVjUZVUlKixsZGfec737nWUwEAzHAuY4zJ9ySmKpFIKBgMKh6P8/oUAMxAk30e5737AADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADW8uR7AlfCGCNJSiQSeZ4JAOBKjD9/jz+fX86MjFR/f78kqbq6Os8zAQBcjcHBQQWDwctun5GRKi8vlyR1d3d/6oO70SUSCVVXV6unp0eBQCDf07EW+2ly2E+Tw36aHGOMBgcHVVVV9anjZmSk3O7sS2nBYJB/BJMQCATYT5PAfpoc9tPksJ8+22QOMrhwAgBgLSIFALDWjIyU3+/X448/Lr/fn++pWI39NDnsp8lhP00O++nacpnPuv4PAIA8mZFHUgCAGwORAgBYi0gBAKxFpAAA1pqRkXrqqac0f/58FRYWasWKFTp06FC+pzStXnvtNX31q19VVVWVXC6XXnrppZztxhht375dN998s4qKilRXV6df/epXOWPOnj2rdevWKRAIKBQKacOGDRoaGprGR3F9NTc3a/ny5ZozZ44qKyu1Zs0adXV15YwZGRlRU1OTKioqVFpaqrVr16q3tzdnTHd3txoaGlRcXKzKyko9/PDDGhsbm86Hcl3t3LlTixcvdv7wNBqN6pVXXnG2s48mtmPHDrlcLm3ZssVZx766TswMs2vXLuPz+cw//uM/muPHj5v777/fhEIh09vbm++pTZt9+/aZv/mbvzH/+q//aiSZPXv25GzfsWOHCQaD5qWXXjLvvPOO+drXvmYWLFhgzp0754y56667zJIlS8ybb75p/uu//svccsst5t57753mR3L91NfXm+eee84cO3bMHD161PzxH/+xqampMUNDQ86YBx54wFRXV5vW1lbz1ltvmZUrV5rf//3fd7aPjY2ZhQsXmrq6OnPkyBGzb98+M3fuXLNt27Z8PKTr4t/+7d/Mv//7v5v/+Z//MV1dXeav//qvjdfrNceOHTPGsI8mcujQITN//nyzePFis3nzZmc9++r6mHGRuvPOO01TU5PzfTqdNlVVVaa5uTmPs8qfiyOVyWRMJBIx3/ve95x1AwMDxu/3mxdffNEYY8yJEyeMJHP48GFnzCuvvGJcLpf54IMPpm3u06mvr89IMm1tbcaY7D7xer1m9+7dzpiTJ08aSaa9vd0Yk/0/A26328RiMWfMzp07TSAQMMlkcnofwDQqKyszzz77LPtoAoODg+bWW281LS0t5ktf+pITKfbV9TOjTveNjo6qo6NDdXV1zjq32626ujq1t7fncWb2OHXqlGKxWM4+CgaDWrFihbOP2tvbFQqFtGzZMmdMXV2d3G63Dh48OO1zng7xeFzSb9+cuKOjQ6lUKmc/3X777aqpqcnZT4sWLVI4HHbG1NfXK5FI6Pjx49M4++mRTqe1a9cuDQ8PKxqNso8m0NTUpIaGhpx9IvHv6XqaUW8w+9FHHymdTuf8jyxJ4XBYv/zlL/M0K7vEYjFJmnAfjW+LxWKqrKzM2e7xeFReXu6MmU0ymYy2bNmiL37xi1q4cKGk7D7w+XwKhUI5Yy/eTxPtx/Fts0VnZ6ei0ahGRkZUWlqqPXv2qLa2VkePHmUfXWDXrl16++23dfjw4Uu28e/p+plRkQKuRFNTk44dO6bXX38931Ox0m233aajR48qHo/rn//5n9XY2Ki2trZ8T8sqPT092rx5s1paWlRYWJjv6dxQZtTpvrlz56qgoOCSK2Z6e3sViUTyNCu7jO+HT9tHkUhEfX19OdvHxsZ09uzZWbcfN23apL179+rAgQOaN2+esz4SiWh0dFQDAwM54y/eTxPtx/Fts4XP59Mtt9yipUuXqrm5WUuWLNEPf/hD9tEFOjo61NfXpzvuuEMej0cej0dtbW168skn5fF4FA6H2VfXyYyKlM/n09KlS9Xa2uqsy2Qyam1tVTQazePM7LFgwQJFIpGcfZRIJHTw4EFnH0WjUQ0MDKijo8MZs3//fmUyGa1YsWLa53w9GGO0adMm7dmzR/v379eCBQtyti9dulRerzdnP3V1dam7uztnP3V2duYEvaWlRYFAQLW1tdPzQPIgk8komUyyjy6watUqdXZ26ujRo86ybNkyrVu3zrnNvrpO8n3lxlTt2rXL+P1+8/zzz5sTJ06Yb37zmyYUCuVcMTPbDQ4OmiNHjpgjR44YSebv/u7vzJEjR8z7779vjMlegh4KhczPfvYz8+6775qvf/3rE16C/nu/93vm4MGD5vXXXze33nrrrLoEfePGjSYYDJpXX33VfPjhh87yySefOGMeeOABU1NTY/bv32/eeustE41GTTQadbaPXzK8evVqc/ToUfPzn//c3HTTTbPqkuFHHnnEtLW1mVOnTpl3333XPPLII8blcpn/+I//MMawjz7NhVf3GcO+ul5mXKSMMebv//7vTU1NjfH5fObOO+80b775Zr6nNK0OHDhgJF2yNDY2GmOyl6E/9thjJhwOG7/fb1atWmW6urpyfkZ/f7+59957TWlpqQkEAua+++4zg4ODeXg018dE+0eSee6555wx586dM3/1V39lysrKTHFxsfnGN75hPvzww5yf895775mvfOUrpqioyMydO9d861vfMqlUapofzfXz53/+5+bzn/+88fl85qabbjKrVq1yAmUM++jTXBwp9tX1wUd1AACsNaNekwIA3FiIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsNb/B3k8kIBR9XtIAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 个动作：-2.0\n",
      "第 2 个动作：-1.6\n",
      "第 3 个动作：-1.2\n",
      "第 4 个动作：-0.8\n",
      "第 5 个动作：-0.3999999999999999\n",
      "第 6 个动作：0.0\n",
      "第 7 个动作：0.3999999999999999\n",
      "第 8 个动作：0.7999999999999998\n",
      "第 9 个动作：1.2000000000000002\n",
      "第 10 个动作：1.6\n",
      "第 11 个动作：2.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(11):\n",
    "    action_continuous = i\n",
    "    action_continuous /=10\n",
    "    action_continuous *=4\n",
    "    action_continuous -= 2\n",
    "    print(f'第 {i+1} 个动作：{action_continuous}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Anaconda install\\envs\\Gym\\lib\\site-packages\\torch\\nn\\modules\\container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[0.0834, 0.0643, 0.0907, 0.0779, 0.0755, 0.1045, 0.0595, 0.0977, 0.1484,\n",
       "          0.0983, 0.0999],\n",
       "         [0.0895, 0.0666, 0.0934, 0.0780, 0.0609, 0.0975, 0.0846, 0.0995, 0.1409,\n",
       "          0.1037, 0.0855],\n",
       "         [0.0846, 0.0748, 0.0880, 0.1535, 0.0759, 0.0878, 0.0430, 0.1208, 0.1183,\n",
       "          0.0761, 0.0773]], grad_fn=<SoftmaxBackward0>),\n",
       " tensor([[-0.5593],\n",
       "         [-0.1584],\n",
       "         [ 0.4558],\n",
       "         [ 0.3665]], grad_fn=<AddmmBackward0>))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 创建两个网络，Actot、Critic\n",
    "import torch\n",
    "model_actor = torch.nn.Sequential(\n",
    "    torch.nn.Linear(3,128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128,11),\n",
    "    torch.nn.Softmax()\n",
    ")\n",
    "model_critic =torch.nn.Sequential(\n",
    "    torch.nn.Linear(3,128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128,1)\n",
    ")\n",
    "model_actor(torch.randn(3,3)),model_critic(torch.randn(4,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0917, 0.0727, 0.0885, 0.0785, 0.0846, 0.1085, 0.0655, 0.1064, 0.1209,\n",
       "         0.0994, 0.0834],\n",
       "        [0.0763, 0.0849, 0.0870, 0.1454, 0.0688, 0.1216, 0.0549, 0.1115, 0.0968,\n",
       "         0.0783, 0.0746],\n",
       "        [0.0874, 0.0877, 0.0965, 0.0634, 0.0728, 0.2181, 0.0416, 0.0944, 0.0951,\n",
       "         0.0761, 0.0669]], grad_fn=<SoftmaxBackward0>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob = model_actor(torch.randn(3,3))\n",
    "prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 1.6)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "\n",
    "    prob = model_actor(state)\n",
    "\n",
    "    action = random.choices(range(11), weights=prob[0].tolist(),k=1)[0]\n",
    "    action_continuous = action\n",
    "    action_continuous /= 10\n",
    "    action_continuous *= 4\n",
    "    action_continuous -= 2\n",
    "\n",
    "    return action, action_continuous\n",
    "\n",
    "\n",
    "get_action([0,1,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action_continuous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.06770282, 0.9977055 , 0.7302374 ], dtype=float32),\n",
       " -2.1508138454134973,\n",
       " False,\n",
       " {})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step([0.1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cgq10\\AppData\\Local\\Temp\\ipykernel_15300\\303743510.py:23: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  states = torch.FloatTensor(states).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-4.6416e-01, -8.8575e-01, -2.1926e-01],\n",
       "         [-4.8980e-01, -8.7183e-01, -5.8358e-01],\n",
       "         [-5.3011e-01, -8.4793e-01, -9.3745e-01],\n",
       "         [-5.9752e-01, -8.0185e-01, -1.6334e+00],\n",
       "         [-6.9192e-01, -7.2197e-01, -2.4748e+00],\n",
       "         [-7.9618e-01, -6.0506e-01, -3.1363e+00],\n",
       "         [-8.8452e-01, -4.6650e-01, -3.2901e+00],\n",
       "         [-9.5612e-01, -2.9298e-01, -3.7599e+00],\n",
       "         [-9.9456e-01, -1.0416e-01, -3.8597e+00],\n",
       "         [-9.9485e-01,  1.0136e-01, -4.1178e+00],\n",
       "         [-9.5245e-01,  3.0470e-01, -4.1618e+00],\n",
       "         [-8.7309e-01,  4.8755e-01, -3.9933e+00],\n",
       "         [-7.6506e-01,  6.4396e-01, -3.8076e+00],\n",
       "         [-6.5479e-01,  7.5581e-01, -3.1446e+00],\n",
       "         [-5.4971e-01,  8.3536e-01, -2.6378e+00],\n",
       "         [-4.7367e-01,  8.8070e-01, -1.7712e+00],\n",
       "         [-4.3219e-01,  9.0178e-01, -9.3071e-01],\n",
       "         [-4.3154e-01,  9.0209e-01, -1.4367e-02],\n",
       "         [-4.6913e-01,  8.8313e-01,  8.4220e-01],\n",
       "         [-5.3164e-01,  8.4697e-01,  1.4446e+00],\n",
       "         [-6.2609e-01,  7.7975e-01,  2.3198e+00],\n",
       "         [-7.3234e-01,  6.8094e-01,  2.9046e+00],\n",
       "         [-8.3245e-01,  5.5410e-01,  3.2353e+00],\n",
       "         [-9.1921e-01,  3.9377e-01,  3.6509e+00],\n",
       "         [-9.7602e-01,  2.1767e-01,  3.7062e+00],\n",
       "         [-9.9966e-01,  2.5949e-02,  3.8695e+00],\n",
       "         [-9.8824e-01, -1.5289e-01,  3.5889e+00],\n",
       "         [-9.5166e-01, -3.0716e-01,  3.1742e+00],\n",
       "         [-9.0157e-01, -4.3262e-01,  2.7039e+00],\n",
       "         [-8.4062e-01, -5.4163e-01,  2.4994e+00],\n",
       "         [-7.8318e-01, -6.2179e-01,  1.9732e+00],\n",
       "         [-7.4225e-01, -6.7012e-01,  1.2668e+00],\n",
       "         [-7.2443e-01, -6.8935e-01,  5.2425e-01],\n",
       "         [-7.1376e-01, -7.0039e-01,  3.0725e-01],\n",
       "         [-7.2135e-01, -6.9257e-01, -2.1805e-01],\n",
       "         [-7.4839e-01, -6.6326e-01, -7.9748e-01],\n",
       "         [-7.8231e-01, -6.2288e-01, -1.0549e+00],\n",
       "         [-8.3076e-01, -5.5663e-01, -1.6421e+00],\n",
       "         [-8.8217e-01, -4.7093e-01, -1.9996e+00],\n",
       "         [-9.3135e-01, -3.6414e-01, -2.3528e+00],\n",
       "         [-9.7242e-01, -2.3324e-01, -2.7459e+00],\n",
       "         [-9.9517e-01, -9.8214e-02, -2.7408e+00],\n",
       "         [-9.9944e-01,  3.3344e-02, -2.6344e+00],\n",
       "         [-9.8561e-01,  1.6901e-01, -2.7294e+00],\n",
       "         [-9.5797e-01,  2.8687e-01, -2.4227e+00],\n",
       "         [-9.1818e-01,  3.9616e-01, -2.3275e+00],\n",
       "         [-8.6739e-01,  4.9762e-01, -2.2704e+00],\n",
       "         [-8.1288e-01,  5.8243e-01, -2.0172e+00],\n",
       "         [-7.5854e-01,  6.5163e-01, -1.7604e+00],\n",
       "         [-7.1140e-01,  7.0279e-01, -1.3917e+00],\n",
       "         [-6.7816e-01,  7.3492e-01, -9.2456e-01],\n",
       "         [-6.7324e-01,  7.3942e-01, -1.3337e-01],\n",
       "         [-6.9731e-01,  7.1676e-01,  6.6119e-01],\n",
       "         [-7.4303e-01,  6.6926e-01,  1.3188e+00],\n",
       "         [-8.0259e-01,  5.9654e-01,  1.8807e+00],\n",
       "         [-8.6644e-01,  4.9928e-01,  2.3281e+00],\n",
       "         [-9.2466e-01,  3.8078e-01,  2.6426e+00],\n",
       "         [-9.7384e-01,  2.2723e-01,  3.2282e+00],\n",
       "         [-9.9842e-01,  5.6276e-02,  3.4586e+00],\n",
       "         [-9.9223e-01, -1.2441e-01,  3.6208e+00],\n",
       "         [-9.5501e-01, -2.9658e-01,  3.5275e+00],\n",
       "         [-8.8775e-01, -4.6033e-01,  3.5451e+00],\n",
       "         [-8.0308e-01, -5.9587e-01,  3.1998e+00],\n",
       "         [-7.2208e-01, -6.9181e-01,  2.5129e+00],\n",
       "         [-6.4735e-01, -7.6220e-01,  2.0540e+00],\n",
       "         [-5.8913e-01, -8.0804e-01,  1.4824e+00],\n",
       "         [-5.5815e-01, -8.2974e-01,  7.5637e-01],\n",
       "         [-5.6006e-01, -8.2845e-01, -4.5937e-02],\n",
       "         [-5.8495e-01, -8.1107e-01, -6.0728e-01],\n",
       "         [-6.2848e-01, -7.7783e-01, -1.0956e+00],\n",
       "         [-6.9796e-01, -7.1614e-01, -1.8589e+00],\n",
       "         [-7.7096e-01, -6.3688e-01, -2.1560e+00],\n",
       "         [-8.5421e-01, -5.1992e-01, -2.8737e+00],\n",
       "         [-9.2507e-01, -3.7980e-01, -3.1437e+00],\n",
       "         [-9.7294e-01, -2.3105e-01, -3.1285e+00],\n",
       "         [-9.9788e-01, -6.5019e-02, -3.3618e+00],\n",
       "         [-9.9535e-01,  9.6313e-02, -3.2306e+00],\n",
       "         [-9.6931e-01,  2.4583e-01, -3.0383e+00],\n",
       "         [-9.2789e-01,  3.7285e-01, -2.6739e+00],\n",
       "         [-8.6941e-01,  4.9409e-01, -2.6943e+00],\n",
       "         [-8.0449e-01,  5.9397e-01, -2.3837e+00],\n",
       "         [-7.3922e-01,  6.7347e-01, -2.0583e+00],\n",
       "         [-6.8474e-01,  7.2879e-01, -1.5532e+00],\n",
       "         [-6.5177e-01,  7.5842e-01, -8.8657e-01],\n",
       "         [-6.4653e-01,  7.6289e-01, -1.3776e-01],\n",
       "         [-6.6743e-01,  7.4467e-01,  5.5440e-01],\n",
       "         [-7.1204e-01,  7.0214e-01,  1.2329e+00],\n",
       "         [-7.8044e-01,  6.2523e-01,  2.0595e+00],\n",
       "         [-8.4512e-01,  5.3457e-01,  2.2284e+00],\n",
       "         [-9.0791e-01,  4.1917e-01,  2.6294e+00],\n",
       "         [-9.5699e-01,  2.9012e-01,  2.7637e+00],\n",
       "         [-9.8947e-01,  1.4477e-01,  2.9813e+00],\n",
       "         [-9.9988e-01, -1.5214e-02,  3.2099e+00],\n",
       "         [-9.8470e-01, -1.7425e-01,  3.1985e+00],\n",
       "         [-9.4456e-01, -3.2833e-01,  3.1878e+00],\n",
       "         [-8.8764e-01, -4.6055e-01,  2.8816e+00],\n",
       "         [-8.1364e-01, -5.8138e-01,  2.8362e+00],\n",
       "         [-7.3818e-01, -6.7460e-01,  2.4001e+00],\n",
       "         [-6.7772e-01, -7.3532e-01,  1.7142e+00],\n",
       "         [-6.2453e-01, -7.8100e-01,  1.4027e+00],\n",
       "         [-5.7996e-01, -8.1464e-01,  1.1169e+00],\n",
       "         [-5.6413e-01, -8.2568e-01,  3.8596e-01],\n",
       "         [-5.7618e-01, -8.1732e-01, -2.9330e-01],\n",
       "         [-6.1498e-01, -7.8854e-01, -9.6629e-01],\n",
       "         [-6.7225e-01, -7.4032e-01, -1.4977e+00],\n",
       "         [-7.5055e-01, -6.6081e-01, -2.2329e+00],\n",
       "         [-8.4002e-01, -5.4255e-01, -2.9686e+00],\n",
       "         [-9.1321e-01, -4.0749e-01, -3.0755e+00],\n",
       "         [-9.6800e-01, -2.5094e-01, -3.3211e+00],\n",
       "         [-9.9620e-01, -8.7072e-02, -3.3293e+00],\n",
       "         [-9.9751e-01,  7.0488e-02, -3.1546e+00],\n",
       "         [-9.7790e-01,  2.0908e-01, -2.8017e+00],\n",
       "         [-9.3664e-01,  3.5029e-01, -2.9449e+00],\n",
       "         [-8.8421e-01,  4.6708e-01, -2.5622e+00],\n",
       "         [-8.3228e-01,  5.5435e-01, -2.0319e+00],\n",
       "         [-7.8109e-01,  6.2442e-01, -1.7361e+00],\n",
       "         [-7.4399e-01,  6.6819e-01, -1.1478e+00],\n",
       "         [-7.1154e-01,  7.0265e-01, -9.4666e-01],\n",
       "         [-6.9015e-01,  7.2366e-01, -5.9968e-01],\n",
       "         [-6.8591e-01,  7.2769e-01, -1.1693e-01],\n",
       "         [-6.9275e-01,  7.2118e-01,  1.8884e-01],\n",
       "         [-7.2895e-01,  6.8457e-01,  1.0297e+00],\n",
       "         [-7.8701e-01,  6.1695e-01,  1.7831e+00],\n",
       "         [-8.5118e-01,  5.2487e-01,  2.2459e+00],\n",
       "         [-9.1285e-01,  4.0830e-01,  2.6395e+00],\n",
       "         [-9.5958e-01,  2.8145e-01,  2.7057e+00],\n",
       "         [-9.9070e-01,  1.3604e-01,  2.9768e+00],\n",
       "         [-9.9998e-01, -5.4815e-03,  2.8388e+00],\n",
       "         [-9.9127e-01, -1.3183e-01,  2.5347e+00],\n",
       "         [-9.7157e-01, -2.3674e-01,  2.1359e+00],\n",
       "         [-9.4378e-01, -3.3059e-01,  1.9583e+00],\n",
       "         [-9.1086e-01, -4.1273e-01,  1.7704e+00],\n",
       "         [-8.8257e-01, -4.7017e-01,  1.2808e+00],\n",
       "         [-8.6286e-01, -5.0544e-01,  8.0819e-01],\n",
       "         [-8.5024e-01, -5.2639e-01,  4.8911e-01],\n",
       "         [-8.5405e-01, -5.2018e-01, -1.4569e-01],\n",
       "         [-8.7359e-01, -4.8667e-01, -7.7582e-01],\n",
       "         [-8.9991e-01, -4.3608e-01, -1.1408e+00],\n",
       "         [-9.2611e-01, -3.7726e-01, -1.2879e+00],\n",
       "         [-9.4819e-01, -3.1770e-01, -1.2708e+00],\n",
       "         [-9.6795e-01, -2.5113e-01, -1.3891e+00],\n",
       "         [-9.8723e-01, -1.5929e-01, -1.8774e+00],\n",
       "         [-9.9761e-01, -6.9070e-02, -1.8169e+00],\n",
       "         [-9.9988e-01,  1.5310e-02, -1.6887e+00],\n",
       "         [-9.9566e-01,  9.3038e-02, -1.5572e+00],\n",
       "         [-9.8600e-01,  1.6676e-01, -1.4875e+00],\n",
       "         [-9.7575e-01,  2.1888e-01, -1.0624e+00],\n",
       "         [-9.6802e-01,  2.5087e-01, -6.5823e-01],\n",
       "         [-9.5936e-01,  2.8219e-01, -6.5007e-01],\n",
       "         [-9.5738e-01,  2.8883e-01, -1.3843e-01],\n",
       "         [-9.5590e-01,  2.9370e-01, -1.0181e-01],\n",
       "         [-9.5499e-01,  2.9664e-01, -6.1537e-02],\n",
       "         [-9.5991e-01,  2.8031e-01,  3.4094e-01],\n",
       "         [-9.6951e-01,  2.4504e-01,  7.3118e-01],\n",
       "         [-9.7910e-01,  2.0339e-01,  8.5496e-01],\n",
       "         [-9.8900e-01,  1.4790e-01,  1.1275e+00],\n",
       "         [-9.9544e-01,  9.5374e-02,  1.0584e+00],\n",
       "         [-9.9935e-01,  3.6014e-02,  1.1899e+00],\n",
       "         [-9.9992e-01, -1.2826e-02,  9.7696e-01],\n",
       "         [-9.9794e-01, -6.4149e-02,  1.0273e+00],\n",
       "         [-9.9291e-01, -1.1887e-01,  1.0992e+00],\n",
       "         [-9.8461e-01, -1.7476e-01,  1.1301e+00],\n",
       "         [-9.7466e-01, -2.2370e-01,  9.9901e-01],\n",
       "         [-9.6292e-01, -2.6979e-01,  9.5123e-01],\n",
       "         [-9.4936e-01, -3.1420e-01,  9.2889e-01],\n",
       "         [-9.3996e-01, -3.4128e-01,  5.7324e-01],\n",
       "         [-9.3119e-01, -3.6454e-01,  4.9728e-01],\n",
       "         [-9.2248e-01, -3.8604e-01,  4.6388e-01],\n",
       "         [-9.1670e-01, -3.9957e-01,  2.9435e-01],\n",
       "         [-9.1071e-01, -4.1304e-01,  2.9467e-01],\n",
       "         [-9.0601e-01, -4.2325e-01,  2.2489e-01],\n",
       "         [-9.1292e-01, -4.0813e-01, -3.3255e-01],\n",
       "         [-9.2088e-01, -3.8985e-01, -3.9865e-01],\n",
       "         [-9.3802e-01, -3.4658e-01, -9.3103e-01],\n",
       "         [-9.6123e-01, -2.7575e-01, -1.4910e+00],\n",
       "         [-9.8285e-01, -1.8442e-01, -1.8778e+00],\n",
       "         [-9.9504e-01, -9.9512e-02, -1.7161e+00],\n",
       "         [-1.0000e+00, -1.1402e-03, -1.9707e+00],\n",
       "         [-9.9526e-01,  9.7285e-02, -1.9716e+00],\n",
       "         [-9.8039e-01,  1.9707e-01, -2.0186e+00],\n",
       "         [-9.5595e-01,  2.9352e-01, -1.9908e+00],\n",
       "         [-9.2283e-01,  3.8522e-01, -1.9507e+00],\n",
       "         [-8.8349e-01,  4.6845e-01, -1.8418e+00],\n",
       "         [-8.3971e-01,  5.4304e-01, -1.7304e+00],\n",
       "         [-8.0908e-01,  5.8770e-01, -1.0831e+00],\n",
       "         [-7.9709e-01,  6.0386e-01, -4.0237e-01],\n",
       "         [-7.9499e-01,  6.0662e-01, -6.9479e-02],\n",
       "         [-7.9758e-01,  6.0322e-01,  8.5487e-02],\n",
       "         [-8.2043e-01,  5.7175e-01,  7.7790e-01],\n",
       "         [-8.5027e-01,  5.2635e-01,  1.0867e+00],\n",
       "         [-8.7986e-01,  4.7523e-01,  1.1815e+00],\n",
       "         [-9.1131e-01,  4.1171e-01,  1.4179e+00],\n",
       "         [-9.4829e-01,  3.1741e-01,  2.0267e+00],\n",
       "         [-9.7808e-01,  2.0822e-01,  2.2647e+00],\n",
       "         [-9.9463e-01,  1.0353e-01,  2.1209e+00],\n",
       "         [-1.0000e+00, -2.1264e-04,  2.0785e+00],\n",
       "         [-9.9326e-01, -1.1587e-01,  2.3184e+00],\n",
       "         [-9.7069e-01, -2.4033e-01,  2.5315e+00],\n",
       "         [-9.3686e-01, -3.4972e-01,  2.2912e+00],\n",
       "         [-8.8988e-01, -4.5620e-01,  2.3289e+00]]),\n",
       " tensor([[ -4.2256],\n",
       "         [ -4.3755],\n",
       "         [ -4.6229],\n",
       "         [ -5.1588],\n",
       "         [ -6.0650],\n",
       "         [ -7.1965],\n",
       "         [ -8.1388],\n",
       "         [ -9.5041],\n",
       "         [-10.7160],\n",
       "         [-10.9382],\n",
       "         [ -9.7522],\n",
       "         [ -8.5251],\n",
       "         [ -7.4142],\n",
       "         [ -6.2089],\n",
       "         [ -5.3329],\n",
       "         [ -4.5763],\n",
       "         [ -4.1603],\n",
       "         [ -4.0697],\n",
       "         [ -4.3110],\n",
       "         [ -4.7538],\n",
       "         [ -5.5886],\n",
       "         [ -6.5694],\n",
       "         [ -7.5712],\n",
       "         [ -8.8258],\n",
       "         [ -9.9127],\n",
       "         [-11.2085],\n",
       "         [-10.2208],\n",
       "         [ -9.0156],\n",
       "         [ -7.9904],\n",
       "         [ -7.2262],\n",
       "         [ -6.4956],\n",
       "         [ -5.9578],\n",
       "         [ -5.7007],\n",
       "         [ -5.6057],\n",
       "         [ -5.6529],\n",
       "         [ -5.9053],\n",
       "         [ -6.2087],\n",
       "         [ -6.7788],\n",
       "         [ -7.4290],\n",
       "         [ -8.2209],\n",
       "         [ -9.2013],\n",
       "         [-10.0138],\n",
       "         [-10.3558],\n",
       "         [ -9.5778],\n",
       "         [ -8.7137],\n",
       "         [ -8.0205],\n",
       "         [ -7.3844],\n",
       "         [ -6.7581],\n",
       "         [ -6.2245],\n",
       "         [ -5.7742],\n",
       "         [ -5.4521],\n",
       "         [ -5.3376],\n",
       "         [ -5.5314],\n",
       "         [ -5.9744],\n",
       "         [ -6.6158],\n",
       "         [ -7.4004],\n",
       "         [ -8.2700],\n",
       "         [ -9.5241],\n",
       "         [-10.7158],\n",
       "         [-10.4124],\n",
       "         [ -9.3152],\n",
       "         [ -8.3495],\n",
       "         [ -7.2927],\n",
       "         [ -6.2846],\n",
       "         [ -5.5970],\n",
       "         [ -5.0638],\n",
       "         [ -4.7370],\n",
       "         [ -4.6887],\n",
       "         [ -4.8582],\n",
       "         [ -5.1857],\n",
       "         [ -5.8394],\n",
       "         [ -6.4755],\n",
       "         [ -7.5596],\n",
       "         [ -8.5658],\n",
       "         [ -9.4379],\n",
       "         [-10.5966],\n",
       "         [-10.3171],\n",
       "         [ -9.2953],\n",
       "         [ -8.3339],\n",
       "         [ -7.6157],\n",
       "         [ -6.8470],\n",
       "         [ -6.1966],\n",
       "         [ -5.6477],\n",
       "         [ -5.2817],\n",
       "         [ -5.1728],\n",
       "         [ -5.3285],\n",
       "         [ -5.7407],\n",
       "         [ -6.5101],\n",
       "         [ -7.1406],\n",
       "         [ -8.0318],\n",
       "         [ -8.8706],\n",
       "         [ -9.8674],\n",
       "         [-10.8046],\n",
       "         [ -9.8235],\n",
       "         [ -8.8960],\n",
       "         [ -7.9258],\n",
       "         [ -7.1607],\n",
       "         [ -6.3431],\n",
       "         [ -5.6578],\n",
       "         [ -5.2422],\n",
       "         [ -4.9192],\n",
       "         [ -4.7247],\n",
       "         [ -4.7823],\n",
       "         [ -5.0805],\n",
       "         [ -5.5528],\n",
       "         [ -6.3561],\n",
       "         [ -7.4805],\n",
       "         [ -8.3547],\n",
       "         [ -9.4446],\n",
       "         [-10.4404],\n",
       "         [-10.4305],\n",
       "         [ -9.3795],\n",
       "         [ -8.6170],\n",
       "         [ -7.7102],\n",
       "         [ -6.9365],\n",
       "         [ -6.3891],\n",
       "         [ -5.9429],\n",
       "         [ -5.6724],\n",
       "         [ -5.4767],\n",
       "         [ -5.4172],\n",
       "         [ -5.4649],\n",
       "         [ -5.8091],\n",
       "         [ -6.4522],\n",
       "         [ -7.2075],\n",
       "         [ -8.1031],\n",
       "         [ -8.8907],\n",
       "         [ -9.9195],\n",
       "         [-10.6451],\n",
       "         [ -9.7028],\n",
       "         [ -8.8812],\n",
       "         [ -8.2498],\n",
       "         [ -7.6923],\n",
       "         [ -7.1984],\n",
       "         [ -6.8864],\n",
       "         [ -6.7203],\n",
       "         [ -6.7362],\n",
       "         [ -6.9946],\n",
       "         [ -7.3696],\n",
       "         [ -7.7585],\n",
       "         [ -8.1049],\n",
       "         [ -8.5361],\n",
       "         [ -9.2440],\n",
       "         [ -9.7716],\n",
       "         [-10.0595],\n",
       "         [ -9.5354],\n",
       "         [ -9.0702],\n",
       "         [ -8.6472],\n",
       "         [ -8.3854],\n",
       "         [ -8.2002],\n",
       "         [ -8.1178],\n",
       "         [ -8.0880],\n",
       "         [ -8.0698],\n",
       "         [ -8.1782],\n",
       "         [ -8.4290],\n",
       "         [ -8.6984],\n",
       "         [ -9.0875],\n",
       "         [ -9.3907],\n",
       "         [ -9.7887],\n",
       "         [ -9.8848],\n",
       "         [ -9.5766],\n",
       "         [ -9.2566],\n",
       "         [ -8.9245],\n",
       "         [ -8.6034],\n",
       "         [ -8.3198],\n",
       "         [ -8.0505],\n",
       "         [ -7.8369],\n",
       "         [ -7.6916],\n",
       "         [ -7.5586],\n",
       "         [ -7.4685],\n",
       "         [ -7.3869],\n",
       "         [ -7.3223],\n",
       "         [ -7.4185],\n",
       "         [ -7.5322],\n",
       "         [ -7.8618],\n",
       "         [ -8.4161],\n",
       "         [ -9.0952],\n",
       "         [ -9.5492],\n",
       "         [-10.2508],\n",
       "         [ -9.6562],\n",
       "         [ -9.0707],\n",
       "         [ -8.4843],\n",
       "         [ -7.9233],\n",
       "         [ -7.3858],\n",
       "         [ -6.8943],\n",
       "         [ -6.4370],\n",
       "         [ -6.2332],\n",
       "         [ -6.2035],\n",
       "         [ -6.2236],\n",
       "         [ -6.4770],\n",
       "         [ -6.8162],\n",
       "         [ -7.1435],\n",
       "         [ -7.5886],\n",
       "         [ -8.3552],\n",
       "         [ -9.1125],\n",
       "         [ -9.6792],\n",
       "         [-10.3029],\n",
       "         [ -9.6949],\n",
       "         [ -9.0445],\n",
       "         [ -8.2814],\n",
       "         [ -7.6606]]),\n",
       " tensor([[10],\n",
       "         [10],\n",
       "         [ 4],\n",
       "         [ 1],\n",
       "         [ 3],\n",
       "         [10],\n",
       "         [ 3],\n",
       "         [ 7],\n",
       "         [ 2],\n",
       "         [ 3],\n",
       "         [ 4],\n",
       "         [ 2],\n",
       "         [ 8],\n",
       "         [ 4],\n",
       "         [ 9],\n",
       "         [ 8],\n",
       "         [ 9],\n",
       "         [ 8],\n",
       "         [ 4],\n",
       "         [ 9],\n",
       "         [ 5],\n",
       "         [ 2],\n",
       "         [ 5],\n",
       "         [ 1],\n",
       "         [ 5],\n",
       "         [ 0],\n",
       "         [ 0],\n",
       "         [ 1],\n",
       "         [ 7],\n",
       "         [ 3],\n",
       "         [ 1],\n",
       "         [ 1],\n",
       "         [10],\n",
       "         [ 5],\n",
       "         [ 4],\n",
       "         [ 9],\n",
       "         [ 3],\n",
       "         [ 6],\n",
       "         [ 5],\n",
       "         [ 3],\n",
       "         [ 8],\n",
       "         [ 8],\n",
       "         [ 3],\n",
       "         [ 8],\n",
       "         [ 3],\n",
       "         [ 1],\n",
       "         [ 3],\n",
       "         [ 2],\n",
       "         [ 3],\n",
       "         [ 4],\n",
       "         [ 9],\n",
       "         [ 9],\n",
       "         [ 7],\n",
       "         [ 6],\n",
       "         [ 5],\n",
       "         [ 4],\n",
       "         [10],\n",
       "         [ 6],\n",
       "         [ 7],\n",
       "         [ 5],\n",
       "         [ 9],\n",
       "         [ 5],\n",
       "         [ 1],\n",
       "         [ 6],\n",
       "         [ 5],\n",
       "         [ 3],\n",
       "         [ 2],\n",
       "         [ 6],\n",
       "         [ 7],\n",
       "         [ 2],\n",
       "         [ 9],\n",
       "         [ 1],\n",
       "         [ 7],\n",
       "         [10],\n",
       "         [ 4],\n",
       "         [ 8],\n",
       "         [ 7],\n",
       "         [ 8],\n",
       "         [ 0],\n",
       "         [ 4],\n",
       "         [ 3],\n",
       "         [ 5],\n",
       "         [ 7],\n",
       "         [ 8],\n",
       "         [ 7],\n",
       "         [ 7],\n",
       "         [10],\n",
       "         [ 0],\n",
       "         [ 5],\n",
       "         [ 2],\n",
       "         [ 5],\n",
       "         [ 7],\n",
       "         [ 5],\n",
       "         [ 7],\n",
       "         [ 4],\n",
       "         [10],\n",
       "         [ 5],\n",
       "         [ 2],\n",
       "         [ 9],\n",
       "         [10],\n",
       "         [ 3],\n",
       "         [ 4],\n",
       "         [ 4],\n",
       "         [ 6],\n",
       "         [ 2],\n",
       "         [ 1],\n",
       "         [10],\n",
       "         [ 6],\n",
       "         [ 8],\n",
       "         [ 9],\n",
       "         [10],\n",
       "         [ 0],\n",
       "         [ 7],\n",
       "         [ 8],\n",
       "         [ 3],\n",
       "         [ 7],\n",
       "         [ 0],\n",
       "         [ 2],\n",
       "         [ 4],\n",
       "         [ 1],\n",
       "         [10],\n",
       "         [ 9],\n",
       "         [ 5],\n",
       "         [ 5],\n",
       "         [ 1],\n",
       "         [ 6],\n",
       "         [ 1],\n",
       "         [ 0],\n",
       "         [ 0],\n",
       "         [ 5],\n",
       "         [ 6],\n",
       "         [ 2],\n",
       "         [ 3],\n",
       "         [ 6],\n",
       "         [ 1],\n",
       "         [ 1],\n",
       "         [ 5],\n",
       "         [ 8],\n",
       "         [10],\n",
       "         [ 7],\n",
       "         [ 0],\n",
       "         [ 8],\n",
       "         [ 8],\n",
       "         [ 7],\n",
       "         [ 5],\n",
       "         [10],\n",
       "         [ 9],\n",
       "         [ 2],\n",
       "         [10],\n",
       "         [ 2],\n",
       "         [ 2],\n",
       "         [ 8],\n",
       "         [ 8],\n",
       "         [ 4],\n",
       "         [ 7],\n",
       "         [ 2],\n",
       "         [ 6],\n",
       "         [ 1],\n",
       "         [ 6],\n",
       "         [ 7],\n",
       "         [ 7],\n",
       "         [ 5],\n",
       "         [ 7],\n",
       "         [ 8],\n",
       "         [ 3],\n",
       "         [ 8],\n",
       "         [ 9],\n",
       "         [ 7],\n",
       "         [10],\n",
       "         [ 9],\n",
       "         [ 1],\n",
       "         [ 9],\n",
       "         [ 1],\n",
       "         [ 0],\n",
       "         [ 2],\n",
       "         [10],\n",
       "         [ 2],\n",
       "         [ 5],\n",
       "         [ 3],\n",
       "         [ 3],\n",
       "         [ 2],\n",
       "         [ 2],\n",
       "         [ 1],\n",
       "         [ 9],\n",
       "         [ 9],\n",
       "         [ 3],\n",
       "         [ 0],\n",
       "         [ 9],\n",
       "         [ 3],\n",
       "         [ 0],\n",
       "         [ 3],\n",
       "         [10],\n",
       "         [ 5],\n",
       "         [ 0],\n",
       "         [ 3],\n",
       "         [ 9],\n",
       "         [10],\n",
       "         [ 4],\n",
       "         [10],\n",
       "         [ 7]]),\n",
       " tensor([[-4.8980e-01, -8.7183e-01, -5.8358e-01],\n",
       "         [-5.3011e-01, -8.4793e-01, -9.3745e-01],\n",
       "         [-5.9752e-01, -8.0185e-01, -1.6334e+00],\n",
       "         [-6.9192e-01, -7.2197e-01, -2.4748e+00],\n",
       "         [-7.9618e-01, -6.0506e-01, -3.1363e+00],\n",
       "         [-8.8452e-01, -4.6650e-01, -3.2901e+00],\n",
       "         [-9.5612e-01, -2.9298e-01, -3.7599e+00],\n",
       "         [-9.9456e-01, -1.0416e-01, -3.8597e+00],\n",
       "         [-9.9485e-01,  1.0136e-01, -4.1178e+00],\n",
       "         [-9.5245e-01,  3.0470e-01, -4.1618e+00],\n",
       "         [-8.7309e-01,  4.8755e-01, -3.9933e+00],\n",
       "         [-7.6506e-01,  6.4396e-01, -3.8076e+00],\n",
       "         [-6.5479e-01,  7.5581e-01, -3.1446e+00],\n",
       "         [-5.4971e-01,  8.3536e-01, -2.6378e+00],\n",
       "         [-4.7367e-01,  8.8070e-01, -1.7712e+00],\n",
       "         [-4.3219e-01,  9.0178e-01, -9.3071e-01],\n",
       "         [-4.3154e-01,  9.0209e-01, -1.4367e-02],\n",
       "         [-4.6913e-01,  8.8313e-01,  8.4220e-01],\n",
       "         [-5.3164e-01,  8.4697e-01,  1.4446e+00],\n",
       "         [-6.2609e-01,  7.7975e-01,  2.3198e+00],\n",
       "         [-7.3234e-01,  6.8094e-01,  2.9046e+00],\n",
       "         [-8.3245e-01,  5.5410e-01,  3.2353e+00],\n",
       "         [-9.1921e-01,  3.9377e-01,  3.6509e+00],\n",
       "         [-9.7602e-01,  2.1767e-01,  3.7062e+00],\n",
       "         [-9.9966e-01,  2.5949e-02,  3.8695e+00],\n",
       "         [-9.8824e-01, -1.5289e-01,  3.5889e+00],\n",
       "         [-9.5166e-01, -3.0716e-01,  3.1742e+00],\n",
       "         [-9.0157e-01, -4.3262e-01,  2.7039e+00],\n",
       "         [-8.4062e-01, -5.4163e-01,  2.4994e+00],\n",
       "         [-7.8318e-01, -6.2179e-01,  1.9732e+00],\n",
       "         [-7.4225e-01, -6.7012e-01,  1.2668e+00],\n",
       "         [-7.2443e-01, -6.8935e-01,  5.2425e-01],\n",
       "         [-7.1376e-01, -7.0039e-01,  3.0725e-01],\n",
       "         [-7.2135e-01, -6.9257e-01, -2.1805e-01],\n",
       "         [-7.4839e-01, -6.6326e-01, -7.9748e-01],\n",
       "         [-7.8231e-01, -6.2288e-01, -1.0549e+00],\n",
       "         [-8.3076e-01, -5.5663e-01, -1.6421e+00],\n",
       "         [-8.8217e-01, -4.7093e-01, -1.9996e+00],\n",
       "         [-9.3135e-01, -3.6414e-01, -2.3528e+00],\n",
       "         [-9.7242e-01, -2.3324e-01, -2.7459e+00],\n",
       "         [-9.9517e-01, -9.8214e-02, -2.7408e+00],\n",
       "         [-9.9944e-01,  3.3344e-02, -2.6344e+00],\n",
       "         [-9.8561e-01,  1.6901e-01, -2.7294e+00],\n",
       "         [-9.5797e-01,  2.8687e-01, -2.4227e+00],\n",
       "         [-9.1818e-01,  3.9616e-01, -2.3275e+00],\n",
       "         [-8.6739e-01,  4.9762e-01, -2.2704e+00],\n",
       "         [-8.1288e-01,  5.8243e-01, -2.0172e+00],\n",
       "         [-7.5854e-01,  6.5163e-01, -1.7604e+00],\n",
       "         [-7.1140e-01,  7.0279e-01, -1.3917e+00],\n",
       "         [-6.7816e-01,  7.3492e-01, -9.2456e-01],\n",
       "         [-6.7324e-01,  7.3942e-01, -1.3337e-01],\n",
       "         [-6.9731e-01,  7.1676e-01,  6.6119e-01],\n",
       "         [-7.4303e-01,  6.6926e-01,  1.3188e+00],\n",
       "         [-8.0259e-01,  5.9654e-01,  1.8807e+00],\n",
       "         [-8.6644e-01,  4.9928e-01,  2.3281e+00],\n",
       "         [-9.2466e-01,  3.8078e-01,  2.6426e+00],\n",
       "         [-9.7384e-01,  2.2723e-01,  3.2282e+00],\n",
       "         [-9.9842e-01,  5.6276e-02,  3.4586e+00],\n",
       "         [-9.9223e-01, -1.2441e-01,  3.6208e+00],\n",
       "         [-9.5501e-01, -2.9658e-01,  3.5275e+00],\n",
       "         [-8.8775e-01, -4.6033e-01,  3.5451e+00],\n",
       "         [-8.0308e-01, -5.9587e-01,  3.1998e+00],\n",
       "         [-7.2208e-01, -6.9181e-01,  2.5129e+00],\n",
       "         [-6.4735e-01, -7.6220e-01,  2.0540e+00],\n",
       "         [-5.8913e-01, -8.0804e-01,  1.4824e+00],\n",
       "         [-5.5815e-01, -8.2974e-01,  7.5637e-01],\n",
       "         [-5.6006e-01, -8.2845e-01, -4.5937e-02],\n",
       "         [-5.8495e-01, -8.1107e-01, -6.0728e-01],\n",
       "         [-6.2848e-01, -7.7783e-01, -1.0956e+00],\n",
       "         [-6.9796e-01, -7.1614e-01, -1.8589e+00],\n",
       "         [-7.7096e-01, -6.3688e-01, -2.1560e+00],\n",
       "         [-8.5421e-01, -5.1992e-01, -2.8737e+00],\n",
       "         [-9.2507e-01, -3.7980e-01, -3.1437e+00],\n",
       "         [-9.7294e-01, -2.3105e-01, -3.1285e+00],\n",
       "         [-9.9788e-01, -6.5019e-02, -3.3618e+00],\n",
       "         [-9.9535e-01,  9.6313e-02, -3.2306e+00],\n",
       "         [-9.6931e-01,  2.4583e-01, -3.0383e+00],\n",
       "         [-9.2789e-01,  3.7285e-01, -2.6739e+00],\n",
       "         [-8.6941e-01,  4.9409e-01, -2.6943e+00],\n",
       "         [-8.0449e-01,  5.9397e-01, -2.3837e+00],\n",
       "         [-7.3922e-01,  6.7347e-01, -2.0583e+00],\n",
       "         [-6.8474e-01,  7.2879e-01, -1.5532e+00],\n",
       "         [-6.5177e-01,  7.5842e-01, -8.8657e-01],\n",
       "         [-6.4653e-01,  7.6289e-01, -1.3776e-01],\n",
       "         [-6.6743e-01,  7.4467e-01,  5.5440e-01],\n",
       "         [-7.1204e-01,  7.0214e-01,  1.2329e+00],\n",
       "         [-7.8044e-01,  6.2523e-01,  2.0595e+00],\n",
       "         [-8.4512e-01,  5.3457e-01,  2.2284e+00],\n",
       "         [-9.0791e-01,  4.1917e-01,  2.6294e+00],\n",
       "         [-9.5699e-01,  2.9012e-01,  2.7637e+00],\n",
       "         [-9.8947e-01,  1.4477e-01,  2.9813e+00],\n",
       "         [-9.9988e-01, -1.5214e-02,  3.2099e+00],\n",
       "         [-9.8470e-01, -1.7425e-01,  3.1985e+00],\n",
       "         [-9.4456e-01, -3.2833e-01,  3.1878e+00],\n",
       "         [-8.8764e-01, -4.6055e-01,  2.8816e+00],\n",
       "         [-8.1364e-01, -5.8138e-01,  2.8362e+00],\n",
       "         [-7.3818e-01, -6.7460e-01,  2.4001e+00],\n",
       "         [-6.7772e-01, -7.3532e-01,  1.7142e+00],\n",
       "         [-6.2453e-01, -7.8100e-01,  1.4027e+00],\n",
       "         [-5.7996e-01, -8.1464e-01,  1.1169e+00],\n",
       "         [-5.6413e-01, -8.2568e-01,  3.8596e-01],\n",
       "         [-5.7618e-01, -8.1732e-01, -2.9330e-01],\n",
       "         [-6.1498e-01, -7.8854e-01, -9.6629e-01],\n",
       "         [-6.7225e-01, -7.4032e-01, -1.4977e+00],\n",
       "         [-7.5055e-01, -6.6081e-01, -2.2329e+00],\n",
       "         [-8.4002e-01, -5.4255e-01, -2.9686e+00],\n",
       "         [-9.1321e-01, -4.0749e-01, -3.0755e+00],\n",
       "         [-9.6800e-01, -2.5094e-01, -3.3211e+00],\n",
       "         [-9.9620e-01, -8.7072e-02, -3.3293e+00],\n",
       "         [-9.9751e-01,  7.0488e-02, -3.1546e+00],\n",
       "         [-9.7790e-01,  2.0908e-01, -2.8017e+00],\n",
       "         [-9.3664e-01,  3.5029e-01, -2.9449e+00],\n",
       "         [-8.8421e-01,  4.6708e-01, -2.5622e+00],\n",
       "         [-8.3228e-01,  5.5435e-01, -2.0319e+00],\n",
       "         [-7.8109e-01,  6.2442e-01, -1.7361e+00],\n",
       "         [-7.4399e-01,  6.6819e-01, -1.1478e+00],\n",
       "         [-7.1154e-01,  7.0265e-01, -9.4666e-01],\n",
       "         [-6.9015e-01,  7.2366e-01, -5.9968e-01],\n",
       "         [-6.8591e-01,  7.2769e-01, -1.1693e-01],\n",
       "         [-6.9275e-01,  7.2118e-01,  1.8884e-01],\n",
       "         [-7.2895e-01,  6.8457e-01,  1.0297e+00],\n",
       "         [-7.8701e-01,  6.1695e-01,  1.7831e+00],\n",
       "         [-8.5118e-01,  5.2487e-01,  2.2459e+00],\n",
       "         [-9.1285e-01,  4.0830e-01,  2.6395e+00],\n",
       "         [-9.5958e-01,  2.8145e-01,  2.7057e+00],\n",
       "         [-9.9070e-01,  1.3604e-01,  2.9768e+00],\n",
       "         [-9.9998e-01, -5.4815e-03,  2.8388e+00],\n",
       "         [-9.9127e-01, -1.3183e-01,  2.5347e+00],\n",
       "         [-9.7157e-01, -2.3674e-01,  2.1359e+00],\n",
       "         [-9.4378e-01, -3.3059e-01,  1.9583e+00],\n",
       "         [-9.1086e-01, -4.1273e-01,  1.7704e+00],\n",
       "         [-8.8257e-01, -4.7017e-01,  1.2808e+00],\n",
       "         [-8.6286e-01, -5.0544e-01,  8.0819e-01],\n",
       "         [-8.5024e-01, -5.2639e-01,  4.8911e-01],\n",
       "         [-8.5405e-01, -5.2018e-01, -1.4569e-01],\n",
       "         [-8.7359e-01, -4.8667e-01, -7.7582e-01],\n",
       "         [-8.9991e-01, -4.3608e-01, -1.1408e+00],\n",
       "         [-9.2611e-01, -3.7726e-01, -1.2879e+00],\n",
       "         [-9.4819e-01, -3.1770e-01, -1.2708e+00],\n",
       "         [-9.6795e-01, -2.5113e-01, -1.3891e+00],\n",
       "         [-9.8723e-01, -1.5929e-01, -1.8774e+00],\n",
       "         [-9.9761e-01, -6.9070e-02, -1.8169e+00],\n",
       "         [-9.9988e-01,  1.5310e-02, -1.6887e+00],\n",
       "         [-9.9566e-01,  9.3038e-02, -1.5572e+00],\n",
       "         [-9.8600e-01,  1.6676e-01, -1.4875e+00],\n",
       "         [-9.7575e-01,  2.1888e-01, -1.0624e+00],\n",
       "         [-9.6802e-01,  2.5087e-01, -6.5823e-01],\n",
       "         [-9.5936e-01,  2.8219e-01, -6.5007e-01],\n",
       "         [-9.5738e-01,  2.8883e-01, -1.3843e-01],\n",
       "         [-9.5590e-01,  2.9370e-01, -1.0181e-01],\n",
       "         [-9.5499e-01,  2.9664e-01, -6.1537e-02],\n",
       "         [-9.5991e-01,  2.8031e-01,  3.4094e-01],\n",
       "         [-9.6951e-01,  2.4504e-01,  7.3118e-01],\n",
       "         [-9.7910e-01,  2.0339e-01,  8.5496e-01],\n",
       "         [-9.8900e-01,  1.4790e-01,  1.1275e+00],\n",
       "         [-9.9544e-01,  9.5374e-02,  1.0584e+00],\n",
       "         [-9.9935e-01,  3.6014e-02,  1.1899e+00],\n",
       "         [-9.9992e-01, -1.2826e-02,  9.7696e-01],\n",
       "         [-9.9794e-01, -6.4149e-02,  1.0273e+00],\n",
       "         [-9.9291e-01, -1.1887e-01,  1.0992e+00],\n",
       "         [-9.8461e-01, -1.7476e-01,  1.1301e+00],\n",
       "         [-9.7466e-01, -2.2370e-01,  9.9901e-01],\n",
       "         [-9.6292e-01, -2.6979e-01,  9.5123e-01],\n",
       "         [-9.4936e-01, -3.1420e-01,  9.2889e-01],\n",
       "         [-9.3996e-01, -3.4128e-01,  5.7324e-01],\n",
       "         [-9.3119e-01, -3.6454e-01,  4.9728e-01],\n",
       "         [-9.2248e-01, -3.8604e-01,  4.6388e-01],\n",
       "         [-9.1670e-01, -3.9957e-01,  2.9435e-01],\n",
       "         [-9.1071e-01, -4.1304e-01,  2.9467e-01],\n",
       "         [-9.0601e-01, -4.2325e-01,  2.2489e-01],\n",
       "         [-9.1292e-01, -4.0813e-01, -3.3255e-01],\n",
       "         [-9.2088e-01, -3.8985e-01, -3.9865e-01],\n",
       "         [-9.3802e-01, -3.4658e-01, -9.3103e-01],\n",
       "         [-9.6123e-01, -2.7575e-01, -1.4910e+00],\n",
       "         [-9.8285e-01, -1.8442e-01, -1.8778e+00],\n",
       "         [-9.9504e-01, -9.9512e-02, -1.7161e+00],\n",
       "         [-1.0000e+00, -1.1402e-03, -1.9707e+00],\n",
       "         [-9.9526e-01,  9.7285e-02, -1.9716e+00],\n",
       "         [-9.8039e-01,  1.9707e-01, -2.0186e+00],\n",
       "         [-9.5595e-01,  2.9352e-01, -1.9908e+00],\n",
       "         [-9.2283e-01,  3.8522e-01, -1.9507e+00],\n",
       "         [-8.8349e-01,  4.6845e-01, -1.8418e+00],\n",
       "         [-8.3971e-01,  5.4304e-01, -1.7304e+00],\n",
       "         [-8.0908e-01,  5.8770e-01, -1.0831e+00],\n",
       "         [-7.9709e-01,  6.0386e-01, -4.0237e-01],\n",
       "         [-7.9499e-01,  6.0662e-01, -6.9479e-02],\n",
       "         [-7.9758e-01,  6.0322e-01,  8.5487e-02],\n",
       "         [-8.2043e-01,  5.7175e-01,  7.7790e-01],\n",
       "         [-8.5027e-01,  5.2635e-01,  1.0867e+00],\n",
       "         [-8.7986e-01,  4.7523e-01,  1.1815e+00],\n",
       "         [-9.1131e-01,  4.1171e-01,  1.4179e+00],\n",
       "         [-9.4829e-01,  3.1741e-01,  2.0267e+00],\n",
       "         [-9.7808e-01,  2.0822e-01,  2.2647e+00],\n",
       "         [-9.9463e-01,  1.0353e-01,  2.1209e+00],\n",
       "         [-1.0000e+00, -2.1264e-04,  2.0785e+00],\n",
       "         [-9.9326e-01, -1.1587e-01,  2.3184e+00],\n",
       "         [-9.7069e-01, -2.4033e-01,  2.5315e+00],\n",
       "         [-9.3686e-01, -3.4972e-01,  2.2912e+00],\n",
       "         [-8.8988e-01, -4.5620e-01,  2.3289e+00],\n",
       "         [-8.3698e-01, -5.4723e-01,  2.1068e+00]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1]]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_data():\n",
    "    states = []\n",
    "    rewards = []\n",
    "    actions = []\n",
    "    next_states = []\n",
    "    overs = []\n",
    "\n",
    "    # 初始化游戏\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, action_continuous = get_action(state)\n",
    "        #print(f'执行动作:{action_continuous}')\n",
    "        next_state, reward, over, _ = env.step([action_continuous])\n",
    "        states.append(state)\n",
    "        rewards.append(reward)\n",
    "        actions.append(action)\n",
    "        next_states.append(next_state)\n",
    "        overs.append(over)\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "    states = torch.FloatTensor(states).reshape(-1, 3)\n",
    "\n",
    "    rewards = torch.FloatTensor(rewards).reshape(-1, 1)\n",
    "\n",
    "    actions = torch.LongTensor(actions).reshape(-1, 1)\n",
    "\n",
    "    next_states = torch.FloatTensor(next_states).reshape(-1, 3)\n",
    "\n",
    "    overs = torch.LongTensor(overs).reshape(-1, 1)\n",
    "\n",
    "    return states, rewards, actions, next_states, overs\n",
    "\n",
    "\n",
    "get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1287.0555839826616"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    \n",
    "    while not over:\n",
    "        \n",
    "        #根据当前状态得到一个动作\n",
    "        action,action_continuous = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action_continuous])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play: \n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0,score:-741.5275327954549\n",
      "epoch:100,score:-798.8885947170604\n",
      "epoch:200,score:-767.4660892896068\n",
      "epoch:300,score:-968.9377520169874\n",
      "epoch:400,score:-867.0402980840136\n",
      "epoch:500,score:-616.861929078009\n",
      "epoch:600,score:-804.1030158531642\n",
      "epoch:700,score:-945.1801382749057\n",
      "epoch:800,score:-691.6443680077499\n",
      "epoch:900,score:-848.6740602117898\n",
      "epoch:1000,score:-711.7229724247597\n",
      "epoch:1100,score:-818.53565622976\n",
      "epoch:1200,score:-801.450393391289\n",
      "epoch:1300,score:-775.6146734517067\n",
      "epoch:1400,score:-624.3354585731083\n",
      "epoch:1500,score:-733.0330511083126\n",
      "epoch:1600,score:-892.1808669827311\n",
      "epoch:1700,score:-728.0230222191103\n",
      "epoch:1800,score:-777.1296268815038\n",
      "epoch:1900,score:-824.8337967597308\n",
      "epoch:2000,score:-785.4000296361398\n",
      "epoch:2100,score:-874.5998464612982\n",
      "epoch:2200,score:-631.2309933475759\n",
      "epoch:2300,score:-226.3882409554123\n",
      "epoch:2400,score:-582.192871915022\n",
      "epoch:2500,score:-1022.5535257458703\n",
      "epoch:2600,score:-554.1967146694599\n",
      "epoch:2700,score:-599.562797026633\n",
      "epoch:2800,score:-739.3535322263242\n",
      "epoch:2900,score:-967.0727626863543\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    optimizer_actor = torch.optim.Adam(model_actor.parameters(), lr=3e-3)\n",
    "    optimizer_critic = torch.optim.Adam(model_critic.parameters(), lr=1e-2)\n",
    "\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    for i in range(3000):\n",
    "        states, rewards, actions, next_states, overs = get_data()\n",
    "\n",
    "        values = model_critic(states)\n",
    "\n",
    "        targets = model_critic(next_states)\n",
    "\n",
    "        targets *= 0.98\n",
    "\n",
    "        targets *= 1 - overs\n",
    "\n",
    "        targets += rewards\n",
    "\n",
    "        deltas = (targets-values).detach()\n",
    "\n",
    "        probs = model_actor(states)\n",
    "\n",
    "        probs = probs.gather(dim=1, index=actions)\n",
    "\n",
    "        loss = (-probs.log() * deltas).mean()\n",
    "\n",
    "        loss_critic =loss_fn(values,targets.detach())\n",
    "\n",
    "        optimizer_actor.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer_actor.step()\n",
    "\n",
    "        optimizer_critic.zero_grad()\n",
    "        loss_critic.backward()\n",
    "        optimizer_critic.step()\n",
    "\n",
    "        if i % 100 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(f\"epoch:{i},score:{test_result}\")\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[9.6347e-01, 3.1397e-02, 1.6002e-04, 1.8970e-04, 3.3033e-05, 9.9847e-05,\n",
       "         4.8105e-05, 2.0001e-04, 4.3145e-03, 6.8350e-05, 1.8126e-05]],\n",
       "       grad_fn=<SoftmaxBackward0>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_actor(torch.tensor([env.reset()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAodElEQVR4nO3de3BUZYL38V93Lh0gdIeA6ciQAI4oRi46XHvdXR3JEDEz3jKla7ksi6yWbGC57OrILGJpWYPFvDuCirBT1oq7NcouKqgM6OYNGpgxXAxEA2iEWSAZoBMQ050Ecunu5/2Dpd9pRe2EJP0kfj9VXSPnnO5++kzTX55zTjoOY4wRAAAWciZ6AAAAfB0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwVsIitXr1ao0YMUJpaWmaMmWKdu/enaihAAAslZBI/ed//qcWL16sxx9/XHv37tX48eNVUFCg+vr6RAwHAGApRyK+YHbKlCmaNGmSnn/+eUlSJBJRTk6O5s+fr0cffbSnhwMAsFRyTz9hW1ubKioqtGTJkugyp9Op/Px8lZeXX/Q+ra2tam1tjf45EonozJkzGjx4sBwOR7ePGQDQtYwxamxs1NChQ+V0fv1BvR6P1OnTpxUOh+X1emOWe71effrppxe9z/Lly/XEE0/0xPAAAD2otrZWw4YN+9r1PR6pzliyZIkWL14c/XMgEFBubq5qa2vldrsTODIAQGcEg0Hl5ORo4MCB37hdj0dqyJAhSkpKUl1dXczyuro6ZWdnX/Q+LpdLLpfrK8vdbjeRAoBe7NtO2fT41X2pqamaMGGCSktLo8sikYhKS0vl8/l6ejgAAIsl5HDf4sWLNWvWLE2cOFGTJ0/WypUr1dzcrNmzZydiOAAASyUkUvfcc49OnTqlZcuWye/367rrrtM777zzlYspAADfbQn5OalLFQwG5fF4FAgEOCcFAL1QvJ/jfHcfAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGt1OFLbt2/XT37yEw0dOlQOh0ObNm2KWW+M0bJly3T55ZerX79+ys/P16FDh2K2OXPmjO677z653W5lZGRozpw5ampquqQXAgDoezocqebmZo0fP16rV6++6PoVK1bo2Wef1dq1a7Vr1y4NGDBABQUFamlpiW5z33336cCBAyopKdHmzZu1fft2Pfjgg51/FQCAvslcAklm48aN0T9HIhGTnZ1tfvnLX0aXNTQ0GJfLZV599VVjjDEHDx40ksyePXui22zdutU4HA5z/PjxuJ43EAgYSSYQCFzK8AEACRLv53iXnpM6cuSI/H6/8vPzo8s8Ho+mTJmi8vJySVJ5ebkyMjI0ceLE6Db5+flyOp3atWvXRR+3tbVVwWAw5gYA6Pu6NFJ+v1+S5PV6Y5Z7vd7oOr/fr6ysrJj1ycnJyszMjG7zZcuXL5fH44necnJyunLYAABL9Yqr+5YsWaJAIBC91dbWJnpIAIAe0KWRys7OliTV1dXFLK+rq4uuy87OVn19fcz6UCikM2fORLf5MpfLJbfbHXMDAPR9XRqpkSNHKjs7W6WlpdFlwWBQu3btks/nkyT5fD41NDSooqIius22bdsUiUQ0ZcqUrhwOAKCXS+7oHZqamnT48OHon48cOaLKykplZmYqNzdXCxcu1FNPPaVRo0Zp5MiReuyxxzR06FDdcccdkqRrrrlGt9xyix544AGtXbtW7e3tmjdvnv7qr/5KQ4cO7bIXBgDoAzp62eB7771nJH3lNmvWLGPM+cvQH3vsMeP1eo3L5TLTpk0z1dXVMY/x+eefm3vvvdekp6cbt9ttZs+ebRobG7v80kUAgJ3i/Rx3GGNMAhvZKcFgUB6PR4FAgPNTANALxfs53iuu7gMAfDcRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtTr8BbMAuleouVlnDx9WS22tQk1NcjidSsnMVL8RI9RvxAg5k/lri+8O3u2AJSKhkBorK1X35pvRQJm2NsnhkDMtTclut9KvvVaX3323XNnZcjg5EIK+j0gBFgg1Nqp+82b5X3tNpr09dqUxipw7p7Zz53Smrk5NBw/qezNnapDPJwezKvRx/FMMSLDwuXOqe+st1W3a9NVAXUSb36/j69YpsHeveuEvMQA6hEgBCWSMUaCiQvVvvqnIuXNx36/t1CkdX7dOrSdPduPogMQjUkAChc+eVc0LLyjS0tLh+7b88Y86+eqrisQx+wJ6KyIFJNCprVs7FagLmg4eVPNnn3XhiAC7ECkggc4eOiQTCnX6/m2nTqn99OkuHBFgFyIFJEjL8eNqO3Uq0cMArEakgAQwxij40Uc6e/hwoocCWI1IAYkSiSR6BID1iBSQCMbIhMOJHgVgPSIFJIIxMsykgG9FpIAEYSYFfDsiBSSCMRKRAr4VkQISwEgc7gPiQKSARODCCSAuRApIFGZSwLciUkAiMJMC4kKkgETgEnQgLkQKSAAjLkEH4kGkgERgJgXEhUgBicJMCvhWRApIBGZSQFyIFJAIfOMEEBciBSQIMyng2xEpIBH4OSkgLkQKSAC+uw+ID5ECEoGZFBAXIgUkAhdOAHEhUkAChJubFdy375Ifp9+IEUrLyemCEQF2IlJAAhhjZEKhS34cR3KyHCkpXTAiwE5ECujNnE7J4Uj0KIBuQ6SAXszhdMrh5K8x+i7e3UBv5nAwk0KfRqSAXoyZFPo63t1Ab8ZMCn0ckQJ6MWZS6Ot4dwO9GVf3oY8jUkAvxkwKfR3vbqA345wU+jgiBfRiDg73oY8jUkBv5nBwuA99Gu9uoBdjJoW+jkgBvRkzKfRxvLuBXszhdJ6/DB3oo3h3A72Z0ykHh/vQhxEpoDfjEnT0cUQK6MU43Ie+jnc30JtxuA99HJECepgxpssey+FwMJNCn8a7G0iESKRrHsfhYCaFPo1IAQlgwuFEDwHoFToUqeXLl2vSpEkaOHCgsrKydMcdd6i6ujpmm5aWFhUXF2vw4MFKT09XUVGR6urqYrapqalRYWGh+vfvr6ysLD388MMKhUKX/mqAXsLwfgfi0qFIlZWVqbi4WDt37lRJSYna29s1ffp0NTc3R7dZtGiR3n77bW3YsEFlZWU6ceKE7rrrruj6cDiswsJCtbW16YMPPtDLL7+sdevWadmyZV33qgDLMZMC4uMwl3AW99SpU8rKylJZWZn+8i//UoFAQJdddpleeeUV/fSnP5Ukffrpp7rmmmtUXl6uqVOnauvWrfrxj3+sEydOyOv1SpLWrl2rn/3sZzp16pRSU1O/9XmDwaA8Ho8CgYDcbndnhw8khDFGzdXVqn7kkUt+rMH5+RrxD//QBaMCela8n+OXdE4qEAhIkjIzMyVJFRUVam9vV35+fnSb0aNHKzc3V+Xl5ZKk8vJyjR07NhooSSooKFAwGNSBAwcu+jytra0KBoMxN6A3YyYFxKfTkYpEIlq4cKFuuOEGjRkzRpLk9/uVmpqqjIyMmG29Xq/8fn90mz8N1IX1F9ZdzPLly+XxeKK3nJyczg4bsAKRAuLT6UgVFxdr//79Wr9+fVeO56KWLFmiQCAQvdXW1nb7cwLdiUgB8UnuzJ3mzZunzZs3a/v27Ro2bFh0eXZ2ttra2tTQ0BAzm6qrq1N2dnZ0m927d8c83oWr/y5s82Uul0sul6szQwWsxNV9QHw6NJMyxmjevHnauHGjtm3bppEjR8asnzBhglJSUlRaWhpdVl1drZqaGvl8PkmSz+dTVVWV6uvro9uUlJTI7XYrLy/vUl4L0HswkwLi0qGZVHFxsV555RW9+eabGjhwYPQcksfjUb9+/eTxeDRnzhwtXrxYmZmZcrvdmj9/vnw+n6ZOnSpJmj59uvLy8jRz5kytWLFCfr9fS5cuVXFxMbMlfGdwuA+IT4citWbNGknSTTfdFLP8pZde0t/+7d9Kkp555hk5nU4VFRWptbVVBQUFeuGFF6LbJiUlafPmzZo7d658Pp8GDBigWbNm6cknn7y0VwL0IkQKiE+HIhXPj1SlpaVp9erVWr169dduM3z4cG3ZsqUjTw30KZyTAuLDd/cBCcBMCogPkQISgEgB8SFSQAIQKSA+nfo5KQCXxoRCMsaoPRLRuXBYreGw2o1ROBKR0+FQVlqaUpOSEj1MIOGIFNDD2tvb9cmxY9px/LiONDbqD42NOnn2rL5obVVjKKTM1FT9y+TJusrjSfRQgYQjUkA3u3BVbFtbm95//3299tpr2vnee6o5dkwhY5Q7YICuGDhQ3qwsZaSmKiM1VVlpaQkeNWAHIgV0I2OMAoGAysvLtXLlSn344YfKyMjQ6IwM3T1mjMZlZirT5VKSwyGnw3H+JPGF/wVApIDuYozRRx99pOeee05vvfWWsrOzNX/+fBUWFmrgli1qrqhI9BAB6xEpoIsZY9Te3q7f/va3+sUvfqGjR4/q7rvv1t/93d8pLy9PqampqvqGH3aPlzMtTenXXNMFIwbsRaSALtbY2KgXXnhBK1euVE5Ojv7jP/5DN998s1JSUuRwOOL65pZ4OJKSlDJoUJc8FmArIgV0oUAgoFWrVmnVqlX64Q9/qMcff1xjxoyRw+HonifkMnX0cUQK6ALGGDU1Nen555/X888/r9tvv11PPPGEvve973VfoBwOOYgU+jgiBXSBcDisl156SStXrtStt96qVatWKT09vfsCJZ2PlJPrANG38Q4HusCOHTu0YsUKTZ06VU888UT3B+p/MZNCX8dMCrgExhgdO3ZMTz31lDIzM7V06VLl5ub2SKDkcEjMpNDHESngEpw7d04vvviiKisrtXr1ak2ePLlnAiXJIWZS6Pv4ZxjQScYYffbZZ3r11VeVn5+v2267rccCJYlzUvhO4B0OdFIkEtGLL76oUCikefPmqV+/fj0+BmZS6OuIFNBJhw4d0uuvv67bbrtN1113Xc8PwOHg56TQ5xEpoBOMMXr99deVnJysW265pceu5vsyDvehr+MdDnSC3+9XaWmpvv/97+vGG29MSKD4YV58FxApoBOqqqr0hz/8QT/+8Y+Vnp6ekDF05uq+L774Qm1tbd0zIKAbECmgg8LhsD799FN98cUXys/PT9xAOvhzUpFIRBs3btShQ4e67Etuge5GpIAOOnv2rD7++GMNGzZMV1xxRULH0pGZlN/v18aNG/Xaa69144iArkWkgA5qbm7WZ599pvHjxys1NbXLH/94c7M219bq1f/5H/3fEyfU3N5+8Q078HNSxhhVV1dr9+7d2rRpk86ePduFIwa6D984AXRQS0uLamtrddNNNympCy9cMMboSFOTHt+3T0ebmtQSDsudkqIxgwbp/0yapJSLBSnO5w+FQiopKdHp06fV2tqqvXv36i/+4i+6bOxAd2EmBXRQW1ub6uvrNXz48C6N1P80NemB3/9enwQCOhcOy0gKtLfr9/X1WrBrlz5vaYm9QwdmUsFgUP/1X/8l6fzhyk2bNikUCnXZ2IHuQqSADopEIgqFQho0aFCXXnq+8sABBb7m0N7u06dVcuLEV5bHe05q27ZtqqmpkXT+wo89e/bo+PHjnR8s0EOIFNAJSUlJl3Q+ypGScsljcMQ5k2pra9OGDRtkjDl/H4dDhw4d0scff8xVfrAekQI66VI+4K9ctqxrBhFHpPbu3auamhrde++9uuaaa+Tz+TR8+HB98MEHavnyIUTAMkQK6IRQKNTpH4p1OBxyJH/1mqXCnBylfM3hwxHp6RqXmdnh5wqFQqqvr9fChQv1T//0T3K5XJo0aZJ+/etfa9CgQVzlB+txdR/QQU6nUykpKfriiy8UiUQ6dfFEisejzB/+UGfeey+6rGDoUEnSUx99pLZwWBFJSQ6HMlJT9S+TJmn4l77ZwltUFNdYb7rpJvXr108HDhxQU1OThgwZojFjxmjYsGFyu90dHjvQk4gU0EGpqanyer06duxYpyPl7NdPg3w+BT78UOHGRknnZ1gFQ4dqWP/+2vzHP+rzlhaNSE/XPSNHarDLFXN/1+WXa5DP960XbjgcDg0cOFCS1NTUpNOnT0d/c/CgQYM6PG6gpxEpoIPS0tKUk5Ojw4cPKxQKKaUTF0E4HA65r79el82YobrXX5cJh6PLxwwapDHfEJDkQYM0dOZMJccxC7oQMWOM6urqdPbsWY0YMSJmHWAzzkkBHTRgwABdddVV+uijj9T+dd8GEQenyyXvnXcq8+abL3qO6mKS0tN1+T33KGPy5A59JVI4HFZlZaW8Xq+8Xm9nhwz0OCIFdFD//v01btw4/fGPf9SRI0cu6bGS+vfXsPvvl7eoSKlZWV+7nSMpSf1GjFDOAw/oshkz5Ozg5e/t7e3avn27xo4dq4yMDGZR6DU43Ad0UFJSkkaPHq3MzEyVlpZq/PjxnX4sh8Oh5AEDdPlPfyr3uHH64oMP1HTggFr9fkVaW5WUnq60YcPkmThRnokT1e9/zyd1lN/v1/79+zVv3jwulkCvQqSAThg3bpyuvPJKbd68WQ8++OAl/04pp8ul9DFjNOCqqxQ+d04mFJKJRORISpIzJUXO/v3ljPOQ4JcZY/TWW29pwIABmjx5crd8KS7QXTjcB3RCVlaWpk2bpsOHD2vHjh1d8s0NDodDTpdLKRkZSh0yRK6sLKUOHqxkt7vTgZKkM2fO6I033tBVV12lyZMnc6gPvQqRAjrB4XCoqKhIoVBIW7ZsUVNTk5VfMWSM0fvvv6/Dhw/r1ltv1ZAhQxI9JKBDiBTQSd///vdVVFSkt956S5WVlYkezldcuOz8N7/5jfr376+//uu/ZhaFXodIAZ3kdDp1//33Kzk5Wc8995zOnTuX6CHFMMZoy5YtKisrU3FxMbMo9EpECugkh8Oh0aNH695771VpaanefPNNaw75GWN0/Phx/epXv9LYsWN1++23M4tCr0SkgEuQlpamBx54QNdff71+8YtfaNeuXYpEIgkdkzFGX3zxhX7+85+rsbFRCxYs0PDhw4kUeiUiBVwCh8Oh4cOHa+nSpWpoaNBTTz2lY8eOJXRGdfbsWa1du1Zbt27V7NmzNWPGDDnj/A2+gG145wJd4M///M/1yCOPaOfOnXrsscfU2NiYkFCFw2G99tpreu6553TjjTdq3rx5SktL6/FxAF2FSAFdICkpSbNnz9aiRYv07rvvat68eaqtre2xQ3/GGDU1Nenf//3f9cgjj2jSpElasWKFBg8e3CPPD3QXvnEC6AIOh0Pp6ekqLi5WKBTSqlWr1NzcrMcff1xjx47t1vNBxhg1NDRo9erVWr16tSZOnKjly5friiuu4DwUej1mUkAXysjI0KJFi/Szn/1Mv//97zV79my98847am1t7fLDf8YYhcNhHT16VMXFxVq5cqWmTp2qZ599Vnl5eQQKfQKRArrYwIEDtWjRIq1Zs0ZJSUmaOXOmFi9erH379nX6V85/WSQSUW1trdauXasZM2Zox44dKi4u1q9//WtmUOhTONwHdDGHw6HU1FTdcccduuKKK/T8889rw4YN2rFjh+68804VFhbq+uuv79QvS7wQp3fffVdvvPGGdu7cqfHjx2vBggWaMWOG+vXr1w2vCEgch7Hlpw87IBgMyuPxKBAI8GsHYDVjjILBoHbt2qWVK1dq9+7dGjhwoK677jrdcccduuGGG5SVlaXk5GQlJyfHXCoeiUQUiUQUCoXU1NSkqqoqvf322yotLdWpU6eUmZmp4uJi3XbbbcrJyeEyc/Qq8X6OEymgm134K9be3q73339fr7/+uvbs2aOjR4+qvb1do0aN0tVXX62cnBwNGTJELpdL4XBYDQ0NOnnypP7whz/owIEDampq0tChQ3XttdfqRz/6ke6+++7oVx1xeA+9DZECLNXa2qrPPvtMVVVVOnjwoPbv36+jR4+qvr5egUBAbW1tcjqdSk9P1+DBg5WTk6NrrrlG1157ra699lp+uy76hHg/xzknBfQwl8ulsWPHasyYMWpra1Nzc7NaWlrU3t6ucDisSCRy/ndLOZ1KSUmRy+VS//79lZaWJqfTSZzwnUKkgARxOBxyuVxyuVyJHgpgLc60AgCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALBWhyK1Zs0ajRs3Tm63W263Wz6fT1u3bo2ub2lpUXFxsQYPHqz09HQVFRWprq4u5jFqampUWFio/v37KysrSw8//LBCoVDXvBoAQJ/SoUgNGzZMTz/9tCoqKvThhx/q5ptv1u23364DBw5IkhYtWqS3335bGzZsUFlZmU6cOKG77rorev9wOKzCwkK1tbXpgw8+0Msvv6x169Zp2bJlXfuqAAB9g7lEgwYNMi+++KJpaGgwKSkpZsOGDdF1n3zyiZFkysvLjTHGbNmyxTidTuP3+6PbrFmzxrjdbtPa2hr3cwYCASPJBAKBSx0+ACAB4v0c7/Q5qXA4rPXr16u5uVk+n08VFRVqb29Xfn5+dJvRo0crNzdX5eXlkqTy8nKNHTtWXq83uk1BQYGCwWB0NnYxra2tCgaDMTcAQN/X4UhVVVUpPT1dLpdLDz30kDZu3Ki8vDz5/X6lpqYqIyMjZnuv1yu/3y9J8vv9MYG6sP7Cuq+zfPlyeTye6C0nJ6ejwwYA9EIdjtTVV1+tyspK7dq1S3PnztWsWbN08ODB7hhb1JIlSxQIBKK32trabn0+AIAdOvwt6KmpqbryyislSRMmTNCePXu0atUq3XPPPWpra1NDQ0PMbKqurk7Z2dmSpOzsbO3evTvm8S5c/Xdhm4vhm6IB4Lvpkn9OKhKJqLW1VRMmTFBKSopKS0uj66qrq1VTUyOfzydJ8vl8qqqqUn19fXSbkpISud1u5eXlXepQAAB9TIdmUkuWLNGMGTOUm5urxsZGvfLKK3r//ff17rvvyuPxaM6cOVq8eLEyMzPldrs1f/58+Xw+TZ06VZI0ffp05eXlaebMmVqxYoX8fr+WLl2q4uJiZkoAgK/oUKTq6+v1N3/zNzp58qQ8Ho/GjRund999Vz/60Y8kSc8884ycTqeKiorU2tqqgoICvfDCC9H7JyUlafPmzZo7d658Pp8GDBigWbNm6cknn+zaVwUA6BMcxhiT6EF0VDAYlMfjUSAQkNvtTvRwAAAdFO/nON/dBwCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBalxSpp59+Wg6HQwsXLowua2lpUXFxsQYPHqz09HQVFRWprq4u5n41NTUqLCxU//79lZWVpYcfflihUOhShgIA6IM6Hak9e/boX//1XzVu3LiY5YsWLdLbb7+tDRs2qKysTCdOnNBdd90VXR8Oh1VYWKi2tjZ98MEHevnll7Vu3TotW7as868CANA3mU5obGw0o0aNMiUlJebGG280CxYsMMYY09DQYFJSUsyGDRui237yySdGkikvLzfGGLNlyxbjdDqN3++PbrNmzRrjdrtNa2trXM8fCASMJBMIBDozfABAgsX7Od6pmVRxcbEKCwuVn58fs7yiokLt7e0xy0ePHq3c3FyVl5dLksrLyzV27Fh5vd7oNgUFBQoGgzpw4MBFn6+1tVXBYDDmBgDo+5I7eof169dr79692rNnz1fW+f1+paamKiMjI2a51+uV3++PbvOngbqw/sK6i1m+fLmeeOKJjg4VANDLdWgmVVtbqwULFug3v/mN0tLSumtMX7FkyRIFAoHorba2tseeGwCQOB2KVEVFherr6/WDH/xAycnJSk5OVllZmZ599lklJyfL6/Wqra1NDQ0NMferq6tTdna2JCk7O/srV/td+POFbb7M5XLJ7XbH3AAAfV+HIjVt2jRVVVWpsrIyeps4caLuu+++6H+npKSotLQ0ep/q6mrV1NTI5/NJknw+n6qqqlRfXx/dpqSkRG63W3l5eV30sgAAfUGHzkkNHDhQY8aMiVk2YMAADR48OLp8zpw5Wrx4sTIzM+V2uzV//nz5fD5NnTpVkjR9+nTl5eVp5syZWrFihfx+v5YuXari4mK5XK4uelkAgL6gwxdOfJtnnnlGTqdTRUVFam1tVUFBgV544YXo+qSkJG3evFlz586Vz+fTgAEDNGvWLD355JNdPRQAQC/nMMaYRA+io4LBoDwejwKBAOenAKAXivdznO/uAwBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYKznRA+gMY4wkKRgMJngkAIDOuPD5feHz/Ov0ykh9/vnnkqScnJwEjwQAcCkaGxvl8Xi+dn2vjFRmZqYkqaam5htf3HddMBhUTk6Oamtr5Xa7Ez0ca7Gf4sN+ig/7KT7GGDU2Nmro0KHfuF2vjJTTef5Umsfj4U0QB7fbzX6KA/spPuyn+LCfvl08kwwunAAAWItIAQCs1Ssj5XK59Pjjj8vlciV6KFZjP8WH/RQf9lN82E9dy2G+7fo/AAASpFfOpAAA3w1ECgBgLSIFALAWkQIAWKtXRmr16tUaMWKE0tLSNGXKFO3evTvRQ+pR27dv109+8hMNHTpUDodDmzZtillvjNGyZct0+eWXq1+/fsrPz9ehQ4ditjlz5ozuu+8+ud1uZWRkaM6cOWpqaurBV9G9li9frkmTJmngwIHKysrSHXfcoerq6phtWlpaVFxcrMGDBys9PV1FRUWqq6uL2aampkaFhYXq37+/srKy9PDDDysUCvXkS+lWa9as0bhx46I/eOrz+bR169boevbRxT399NNyOBxauHBhdBn7qpuYXmb9+vUmNTXV/Nu//Zs5cOCAeeCBB0xGRoapq6tL9NB6zJYtW8w///M/mzfeeMNIMhs3boxZ//TTTxuPx2M2bdpkPvroI3PbbbeZkSNHmnPnzkW3ueWWW8z48ePNzp07zY4dO8yVV15p7r333h5+Jd2noKDAvPTSS2b//v2msrLS3HrrrSY3N9c0NTVFt3nooYdMTk6OKS0tNR9++KGZOnWq+bM/+7Po+lAoZMaMGWPy8/PNvn37zJYtW8yQIUPMkiVLEvGSusVbb71lfvvb35rPPvvMVFdXm5///OcmJSXF7N+/3xjDPrqY3bt3mxEjRphx48aZBQsWRJezr7pHr4vU5MmTTXFxcfTP4XDYDB061CxfvjyBo0qcL0cqEomY7Oxs88tf/jK6rKGhwbhcLvPqq68aY4w5ePCgkWT27NkT3Wbr1q3G4XCY48eP99jYe1J9fb2RZMrKyowx5/dJSkqK2bBhQ3SbTz75xEgy5eXlxpjz/xhwOp3G7/dHt1mzZo1xu92mtbW1Z19ADxo0aJB58cUX2UcX0djYaEaNGmVKSkrMjTfeGI0U+6r79KrDfW1tbaqoqFB+fn50mdPpVH5+vsrLyxM4MnscOXJEfr8/Zh95PB5NmTIluo/Ky8uVkZGhiRMnRrfJz8+X0+nUrl27enzMPSEQCEj6/19OXFFRofb29pj9NHr0aOXm5sbsp7Fjx8rr9Ua3KSgoUDAY1IEDB3pw9D0jHA5r/fr1am5uls/nYx9dRHFxsQoLC2P2icT7qTv1qi+YPX36tMLhcMz/yZLk9Xr16aefJmhUdvH7/ZJ00X10YZ3f71dWVlbM+uTkZGVmZka36UsikYgWLlyoG264QWPGjJF0fh+kpqYqIyMjZtsv76eL7ccL6/qKqqoq+Xw+tbS0KD09XRs3blReXp4qKyvZR39i/fr12rt3r/bs2fOVdbyfuk+vihTQGcXFxdq/f79+97vfJXooVrr66qtVWVmpQCCg1157TbNmzVJZWVmih2WV2tpaLViwQCUlJUpLS0v0cL5TetXhviFDhigpKekrV8zU1dUpOzs7QaOyy4X98E37KDs7W/X19THrQ6GQzpw50+f247x587R582a99957GjZsWHR5dna22tra1NDQELP9l/fTxfbjhXV9RWpqqq688kpNmDBBy5cv1/jx47Vq1Sr20Z+oqKhQfX29fvCDHyg5OVnJyckqKyvTs88+q+TkZHm9XvZVN+lVkUpNTdWECRNUWloaXRaJRFRaWiqfz5fAkdlj5MiRys7OjtlHwWBQu3btiu4jn8+nhoYGVVRURLfZtm2bIpGIpkyZ0uNj7g7GGM2bN08bN27Utm3bNHLkyJj1EyZMUEpKSsx+qq6uVk1NTcx+qqqqigl6SUmJ3G638vLyeuaFJEAkElFrayv76E9MmzZNVVVVqqysjN4mTpyo++67L/rf7KtukugrNzpq/fr1xuVymXXr1pmDBw+aBx980GRkZMRcMdPXNTY2mn379pl9+/YZSeZXv/qV2bdvnzl27Jgx5vwl6BkZGebNN980H3/8sbn99tsvegn69ddfb3bt2mV+97vfmVGjRvWpS9Dnzp1rPB6Pef/9983Jkyejt7Nnz0a3eeihh0xubq7Ztm2b+fDDD43P5zM+ny+6/sIlw9OnTzeVlZXmnXfeMZdddlmfumT40UcfNWVlZebIkSPm448/No8++qhxOBzmv//7v40x7KNv8qdX9xnDvuouvS5Sxhjz3HPPmdzcXJOammomT55sdu7cmegh9aj33nvPSPrKbdasWcaY85ehP/bYY8br9RqXy2WmTZtmqqurYx7j888/N/fee69JT083brfbzJ492zQ2Nibg1XSPi+0fSeall16KbnPu3Dnz93//92bQoEGmf//+5s477zQnT56MeZyjR4+aGTNmmH79+pkhQ4aYf/zHfzTt7e09/Gq6z/3332+GDx9uUlNTzWWXXWamTZsWDZQx7KNv8uVIsa+6B7+qAwBgrV51TgoA8N1CpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLX+H6HxPLbzehZ+AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-116.61761661134608"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Gym",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
